From d7006e03ad79ea30235cc84cbe8ff8b77b2fae42 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:42:34 +0000 Subject: [PATCH 001/138] feat: checkpoint native orchestration backend Preserve in-progress storage, scheduling, claims, credentials and native API/CLI work. The orchestration backend remains incomplete and is not ready for delivery. --- Cargo.lock | 2125 ++++++++++++++++- Cargo.toml | 18 + crates/sqlbuild-cli/Cargo.toml | 5 + .../sqlbuild-cli/src/argv/command_table.json | 139 +- .../src/launcher/_helpers/dispatch.rs | 19 +- .../src/native_commands/_helpers/decision.rs | 4 +- .../src/native_commands/constants.rs | 2 + .../src/native_commands/main/_run_control.rs | 135 ++ .../src/native_commands/main/mod.rs | 3 + .../main/parse_native_invocation.rs | 49 + .../main/run_control_invocation.rs | 63 + .../main/run_native_invocation.rs | 41 +- .../src/native_commands/models.rs | 12 + crates/sqlbuild-cli/tests/native_commands.rs | 6 + .../tests/native_commands/helpers.rs | 108 + .../tests/native_commands/test_control.rs | 100 + .../tests/native_commands/test_types.rs | 15 + crates/sqlbuild-orchestration/Cargo.toml | 36 + .../src/api/_helpers/authentication.rs | 38 + .../src/api/_helpers/mod.rs | 3 + .../src/api/_helpers/runs.rs | 65 + .../src/api/_helpers/snapshots.rs | 65 + .../sqlbuild-orchestration/src/api/errors.rs | 54 + .../src/api/main/mod.rs | 2 + .../src/api/main/openapi_document.rs | 19 + .../src/api/main/serve_api.rs | 39 + crates/sqlbuild-orchestration/src/api/mod.rs | 6 + .../sqlbuild-orchestration/src/api/models.rs | 24 + .../src/credentials/_helpers/audit.rs | 31 + .../src/credentials/_helpers/mod.rs | 2 + .../src/credentials/_helpers/scopes.rs | 40 + .../src/credentials/constants.rs | 4 + .../credentials/main/authenticate_token.rs | 59 + .../src/credentials/main/create_principal.rs | 55 + .../src/credentials/main/issue_token.rs | 78 + .../src/credentials/main/mod.rs | 4 + .../src/credentials/main/revoke_token.rs | 48 + .../src/credentials/mod.rs | 6 + .../src/credentials/models.rs | 63 + .../src/declarations/main/change_trigger.rs | 87 + .../declarations/main/list_trigger_events.rs | 28 + .../src/declarations/main/list_triggers.rs | 30 + .../src/declarations/main/mod.rs | 5 + .../declarations/main/publish_definitions.rs | 94 + .../declarations/main/snapshot_definitions.rs | 79 + .../src/declarations/mod.rs | 2 + .../src/declarations/models.rs | 139 ++ crates/sqlbuild-orchestration/src/lib.rs | 11 + .../src/runs/_helpers/changes.rs | 120 + .../src/runs/_helpers/claims.rs | 84 + .../src/runs/_helpers/events.rs | 63 + .../src/runs/_helpers/mod.rs | 5 + .../src/runs/_helpers/nodes.rs | 83 + .../src/runs/_helpers/projection.rs | 115 + .../src/runs/main/change_run.rs | 104 + .../src/runs/main/claim_next_run.rs | 94 + .../src/runs/main/heartbeat_run.rs | 71 + .../src/runs/main/list_run_nodes.rs | 29 + .../src/runs/main/list_runs.rs | 30 + .../src/runs/main/mod.rs | 7 + .../src/runs/main/queue_run.rs | 57 + .../src/runs/main/request_run.rs | 67 + crates/sqlbuild-orchestration/src/runs/mod.rs | 3 + .../sqlbuild-orchestration/src/runs/models.rs | 250 ++ .../src/scheduler/_helpers/calendar.rs | 31 + .../src/scheduler/_helpers/expansion.rs | 58 + .../src/scheduler/_helpers/mod.rs | 4 + .../src/scheduler/_helpers/persistence.rs | 76 + .../src/scheduler/_helpers/ticks.rs | 38 + .../src/scheduler/constants.rs | 8 + .../src/scheduler/main/acquire_lease.rs | 37 + .../src/scheduler/main/check_lease.rs | 22 + .../src/scheduler/main/evaluate_trigger.rs | 81 + .../src/scheduler/main/list_ticks.rs | 29 + .../src/scheduler/main/mod.rs | 12 + .../src/scheduler/main/next_cron_time.rs | 28 + .../src/scheduler/main/next_schedule_time.rs | 28 + .../src/scheduler/main/parse_cron.rs | 24 + .../src/scheduler/main/plan_ticks.rs | 84 + .../src/scheduler/main/previous_cron_time.rs | 30 + .../src/scheduler/main/renew_lease.rs | 30 + .../src/scheduler/main/run_scheduler.rs | 26 + .../src/scheduler/main/scheduler_cycle.rs | 75 + .../src/scheduler/mod.rs | 9 + .../src/scheduler/models.rs | 150 ++ .../src/scheduler/tests.rs | 12 + .../src/scheduler/tests/helpers.rs | 129 + .../src/scheduler/tests/test_backlog.rs | 107 + .../src/scheduler/tests/test_cron.rs | 158 ++ .../src/scheduler/tests/test_simulation.rs | 32 + .../src/scheduler/tests/test_ticks.rs | 146 ++ .../src/scheduler/tests/test_types.rs | 52 + .../src/snapshots/_helpers/content.rs | 72 + .../src/snapshots/_helpers/mod.rs | 4 + .../src/snapshots/_helpers/preparation.rs | 74 + .../src/snapshots/_helpers/publication.rs | 34 + .../src/snapshots/_helpers/reading.rs | 57 + .../src/snapshots/main/current_snapshot.rs | 53 + .../src/snapshots/main/mod.rs | 2 + .../src/snapshots/main/publish_snapshot.rs | 118 + .../src/snapshots/mod.rs | 3 + .../src/snapshots/models.rs | 79 + .../src/store/_helpers/database.rs | 154 ++ .../src/store/_helpers/migration.rs | 13 + .../src/store/_helpers/mod.rs | 5 + .../src/store/_helpers/open_sqlite.rs | 37 + .../src/store/_helpers/recall_rows.rs | 111 + .../src/store/_helpers/recall_schema.rs | 13 + .../src/store/errors.rs | 22 + .../src/store/main/append_recall.rs | 36 + .../src/store/main/begin.rs | 59 + .../src/store/main/commit.rs | 18 + .../src/store/main/create_location.rs | 30 + .../src/store/main/decode_row.rs | 12 + .../src/store/main/migrate_store.rs | 70 + .../src/store/main/mod.rs | 12 + .../src/store/main/open_store.rs | 23 + .../src/store/main/publish_recall.rs | 25 + .../src/store/main/query.rs | 35 + .../src/store/main/read_recall_events.rs | 28 + .../src/store/main/read_recall_view.rs | 46 + .../src/store/main/refresh_clock.rs | 26 + .../sqlbuild-orchestration/src/store/mod.rs | 5 + .../src/store/models.rs | 59 + .../src/store/schema.sql | 264 ++ .../sqlbuild-orchestration/src/store/types.rs | 3 + crates/sqlbuild-orchestration/tests/api.rs | 12 + .../tests/api/helpers.rs | 52 + .../tests/api/test_contract.rs | 331 +++ .../tests/api/test_types.rs | 7 + .../tests/credentials.rs | 12 + .../tests/credentials/test_contract.rs | 241 ++ crates/sqlbuild-orchestration/tests/recall.rs | 12 + .../tests/recall/helpers.rs | 137 ++ .../tests/recall/test_contract.rs | 28 + .../tests/recall/test_types.rs | 13 + .../tests/recall/test_views.rs | 192 ++ crates/sqlbuild-orchestration/tests/runs.rs | 18 + .../tests/runs/helpers.rs | 118 + .../tests/runs/test_claims.rs | 252 ++ .../tests/runs/test_coalesce.rs | 103 + .../tests/runs/test_faults.rs | 162 ++ .../tests/runs/test_lifecycle.rs | 427 ++++ .../sqlbuild-orchestration/tests/scheduler.rs | 10 + .../tests/scheduler/helpers.rs | 91 + .../tests/scheduler/test_contract.rs | 28 + .../tests/scheduler/test_types.rs | 10 + .../sqlbuild-orchestration/tests/snapshots.rs | 10 + .../tests/snapshots/helpers.rs | 294 +++ .../tests/snapshots/test_contract.rs | 48 + .../tests/snapshots/test_types.rs | 22 + .../tests/store/helpers.rs | 96 + .../tests/store/test_types.rs | 16 + crates/sqlbuild-orchestration/tests/ticks.rs | 16 + .../tests/ticks/fault_helpers.rs | 66 + .../tests/ticks/helpers.rs | 240 ++ .../tests/ticks/test_contract.rs | 32 + .../tests/ticks/test_loop.rs | 98 + .../tests/ticks/test_triggers.rs | 195 ++ .../tests/ticks/test_types.rs | 23 + crates/sqlbuild-recall/Cargo.toml | 1 + .../src/clickhouse/_helpers/literals.rs | 7 +- .../src/records/_helpers/rows.rs | 11 + .../src/schema/_helpers/mod.rs | 4 +- .../src/schema/_helpers/postgres_views.rs | 91 + .../src/schema/_helpers/relational_tables.rs | 69 + .../{sqlite_views.rs => relational_views.rs} | 43 +- .../schema/main/control_table_statements.rs | 14 + .../schema/main/control_view_statements.rs | 34 + crates/sqlbuild-recall/src/schema/main/mod.rs | 3 + .../main/postgres_partition_statement.rs | 24 + .../src/schema/main/table_statements.rs | 40 +- .../src/schema/main/view_statements.rs | 4 +- .../sqlbuild-recall/src/store/_helpers/mod.rs | 1 + .../src/store/_helpers/relational.rs | 12 + .../src/store/_helpers/rows.rs | 79 +- .../src/store/main/conflict_evidence.rs | 23 + crates/sqlbuild-recall/src/store/main/mod.rs | 3 + .../src/store/main/relational_insert.rs | 48 + .../src/store/main/relational_lookup.rs | 48 + crates/sqlbuild-recall/src/store/models.rs | 9 +- fensu.toml | 1 + 182 files changed, 12144 insertions(+), 197 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_control.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/helpers.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/test_control.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/test_types.rs create mode 100644 crates/sqlbuild-orchestration/Cargo.toml create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/runs.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs create mode 100644 crates/sqlbuild-orchestration/src/api/errors.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/openapi_document.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/serve_api.rs create mode 100644 crates/sqlbuild-orchestration/src/api/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/models.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/models.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/models.rs create mode 100644 crates/sqlbuild-orchestration/src/lib.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/events.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/change_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_runs.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/queue_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/models.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/models.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/database.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/migration.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs create mode 100644 crates/sqlbuild-orchestration/src/store/errors.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/append_recall.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/begin.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/commit.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/create_location.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/decode_row.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/migrate_store.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/open_store.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/publish_recall.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/query.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs create mode 100644 crates/sqlbuild-orchestration/src/store/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/schema.sql create mode 100644 crates/sqlbuild-orchestration/src/store/types.rs create mode 100644 crates/sqlbuild-orchestration/tests/api.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/recall/test_views.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_claims.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_faults.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_loop.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs create mode 100644 crates/sqlbuild-orchestration/tests/ticks/test_types.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs rename crates/sqlbuild-recall/src/schema/_helpers/{sqlite_views.rs => relational_views.rs} (92%) create mode 100644 crates/sqlbuild-recall/src/schema/main/control_table_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/control_view_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/relational.rs create mode 100644 crates/sqlbuild-recall/src/store/main/conflict_evidence.rs create mode 100644 crates/sqlbuild-recall/src/store/main/relational_insert.rs create mode 100644 crates/sqlbuild-recall/src/store/main/relational_lookup.rs diff --git a/Cargo.lock b/Cargo.lock index 5e2d5968a1..cef38f434c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -32,6 +53,21 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "attribute-derive" version = "0.10.5" @@ -68,6 +104,85 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -106,6 +221,30 @@ dependencies = [ "objc2", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "bstr" version = "1.13.1" @@ -117,6 +256,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "castaway" version = "0.2.4" @@ -148,6 +305,41 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "collection_literals" version = "1.0.3" @@ -184,6 +376,18 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -202,6 +406,41 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croner" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -221,6 +460,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -245,7 +493,42 @@ checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", "nix 0.31.3", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", ] [[package]] @@ -259,6 +542,37 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -267,6 +581,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -281,6 +596,23 @@ dependencies = [ "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" version = "1.18.0" @@ -300,7 +632,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", ] [[package]] @@ -340,12 +693,98 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -397,8 +836,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", ] [[package]] @@ -408,8 +861,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -431,6 +887,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -456,13 +914,287 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "indexmap" -version = "2.14.2" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.11", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -480,6 +1212,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is-macro" version = "0.3.8" @@ -507,6 +1245,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "junction" version = "2.1.0" @@ -514,7 +1263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" dependencies = [ "scopeguard", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -532,6 +1281,18 @@ dependencies = [ "cc", ] +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.5", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -549,12 +1310,33 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "manyhow" version = "0.11.4" @@ -578,6 +1360,22 @@ dependencies = [ "quote", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -602,6 +1400,33 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nix" version = "0.27.1" @@ -625,6 +1450,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "objc2" version = "0.6.5" @@ -664,13 +1498,57 @@ dependencies = [ "indexmap", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "phf" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", ] [[package]] @@ -680,7 +1558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.11.3", ] [[package]] @@ -689,8 +1567,8 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", - "rand", + "phf_shared 0.11.3", + "rand 0.8.8", ] [[package]] @@ -702,12 +1580,33 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -744,6 +1643,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -753,6 +1661,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -773,6 +1690,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "chacha20", + "core_detect", + "num-traits", + "rand 0.10.3", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "psm" version = "0.1.32" @@ -845,7 +1782,69 @@ dependencies = [ ] [[package]] -name = "quote" +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" @@ -875,6 +1874,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -889,7 +1894,18 @@ checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -899,7 +1915,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -911,6 +1927,30 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.12.0" @@ -931,6 +1971,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -960,6 +2018,58 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.11", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "ruff_python_ast" version = "0.0.4" @@ -1065,7 +2175,42 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1074,6 +2219,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -1105,6 +2262,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -1149,6 +2312,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -1158,6 +2332,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -1195,17 +2387,42 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] [[package]] name = "sqlbuild-analysis" @@ -1252,7 +2469,7 @@ dependencies = [ "syn 2.0.119", "syn 3.0.3", "thiserror 2.0.21", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1275,6 +2492,8 @@ dependencies = [ "blake3", "ctrlc", "getrandom 0.4.3", + "reqwest", + "rusqlite", "rustix", "serde", "serde_json", @@ -1284,8 +2503,11 @@ dependencies = [ "sqlbuild-core", "sqlbuild-cursor", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-rules", "tempfile", + "tokio", + "uuid", ] [[package]] @@ -1343,6 +2565,33 @@ dependencies = [ "sqlbuild-core", ] +[[package]] +name = "sqlbuild-orchestration" +version = "0.186.2" +dependencies = [ + "axum", + "chrono", + "chrono-tz", + "croner", + "flate2", + "getrandom 0.4.3", + "proptest", + "reqwest", + "rusqlite", + "semver", + "serde", + "serde_json", + "sqlbuild-recall", + "sqlx-core", + "sqlx-postgres", + "subtle", + "tempfile", + "thiserror 2.0.21", + "tokio", + "utoipa", + "uuid", +] + [[package]] name = "sqlbuild-project" version = "0.186.2" @@ -1403,6 +2652,7 @@ name = "sqlbuild-recall" version = "0.186.2" dependencies = [ "blake3", + "chrono", "rusqlite", "rustix", "serde_json", @@ -1495,6 +2745,87 @@ dependencies = [ "sqlbuild-cursor", ] +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "smallvec", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.8", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.21", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "stacker" version = "0.1.25" @@ -1505,7 +2836,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1514,6 +2845,50 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -1536,6 +2911,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -1552,7 +2947,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1601,6 +2996,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.13.3" @@ -1608,25 +3013,73 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] -name = "toml" -version = "0.9.12+spec-1.1.0" +name = "tokio" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime 0.7.5+spec-1.1.0", - "toml_parser", - "toml_writer", - "winnow 0.7.15", + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", ] [[package]] -name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +name = "tokio-macros" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] @@ -1667,12 +3120,107 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1688,6 +3236,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -1706,7 +3260,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" dependencies = [ - "phf", + "phf 0.11.3", "unicode_names2_generator", ] @@ -1719,7 +3273,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand", + "rand 0.8.8", ] [[package]] @@ -1728,6 +3282,76 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "uuid", +] + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "sha1_smol", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -1740,6 +3364,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -1750,12 +3383,139 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.3", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.11", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1778,7 +3538,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1787,12 +3547,83 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1802,6 +3633,127 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -1817,6 +3769,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.62" @@ -1837,6 +3824,72 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 6014fffc19..ea73843c9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/sqlbuild-config", "crates/sqlbuild-host", "crates/sqlbuild-recall", + "crates/sqlbuild-orchestration", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", @@ -39,6 +40,7 @@ order = [ "sqlbuild-config", "sqlbuild-host", "sqlbuild-recall", + "sqlbuild-orchestration", "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", @@ -62,6 +64,7 @@ publish = false rust-version = "1.88" [workspace.dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio"] } fensu-policy = "=0.22.0" # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" @@ -115,10 +118,23 @@ rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } # Every crate relies on insertion-ordered JSON objects, which the extension has always built with. serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" +chrono = { version = "0.4", features = ["serde"] } +chrono-tz = "0.10" +croner = "=3.0.1" +flate2 = "1" +proptest = "1" +semver = "1" +# Pin SQLx's driver crates together: its umbrella also resolves an incompatible SQLite engine. +sqlx-core = { version = "=0.8.6", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "uuid", "json"] } +sqlx-postgres = { version = "=0.8.6", default-features = false, features = ["chrono", "uuid", "json"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "signal", "process", "net"] } +uuid = { version = "=1.18.1", features = ["v5", "v7", "serde"] } +utoipa = { version = "5", features = ["chrono", "uuid"] } sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } sqlbuild-attachments = { path = "crates/sqlbuild-attachments" } sqlbuild-build-features = { path = "crates/sqlbuild-build-features" } @@ -133,6 +149,7 @@ sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } +sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } @@ -141,6 +158,7 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } +subtle = "2" tempfile = "3" toml = "0.9" toml_edit = { version = "0.25", default-features = false, features = ["parse"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 4a2253fb56..686080a599 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -26,7 +26,10 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-rules.workspace = true +tokio.workspace = true +uuid.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true @@ -35,6 +38,8 @@ rustix.workspace = true ctrlc.workspace = true [dev-dependencies] +reqwest.workspace = true +rusqlite.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d2b8b61be1..ec58647f34 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,133 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "server", + "parser": { + "prog": "sqb server", + "description": "Run the native control API and scheduler without importing project Python.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--host" + ], + "dest": "control_host", + "default": "127.0.0.1", + "help": "IP address to listen on (default: 127.0.0.1)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--port" + ], + "dest": "control_port", + "default": 8000, + "type": "int", + "help": "TCP port, or 0 to choose a free port (default: 8000)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-scheduler" + ], + "dest": "control_no_scheduler", + "nargs": 0, + "const": true, + "default": false, + "help": "serve the API without the embedded scheduler", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--openapi" + ], + "dest": "control_openapi", + "nargs": 0, + "const": true, + "default": false, + "help": "print the OpenAPI contract as JSON without opening a database", + "group": 1 + } + ] + } + }, + { + "name": "scheduler", + "parser": { + "prog": "sqb scheduler", + "description": "Run the lease-guarded native scheduler against the server's control database.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + } + ] + } + }, { "name": "compile", "parser": { @@ -7592,6 +7719,16 @@ } ], "choices_help": [ + { + "dest": "server", + "metavar": "server", + "help": "run the native control API and scheduler" + }, + { + "dest": "scheduler", + "metavar": "scheduler", + "help": "run a standalone lease-guarded scheduler" + }, { "dest": "cost", "metavar": "cost", @@ -7620,4 +7757,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index e6f689c242..441901c2b6 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -10,7 +10,10 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::parse_native_invocation::parse_native_invocation; +use crate::native_commands::main::run_control_invocation::run_control_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; +use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -38,6 +41,15 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - print_version(); return 0; } + let native = parse_native_invocation(&arguments); + if let Ok(NativeParse::Exit(code)) = &native { + return *code; + } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_control_invocation(namespace) + { + return code; + } let program: PathBuf = std::env::current_exe().unwrap_or_default(); let Some(script) = python_cli(&program) else { report_missing_python_cli(&program); @@ -50,7 +62,12 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - { return exit_code(code); } - if let Ok(code) = run_native_invocation(&arguments, &script) { + if let Ok(NativeParse::Invocation { + namespace, + use_color, + }) = native + && let Ok(code) = run_native_invocation(&namespace, &script, use_color) + { return code; } if let Some((path, _)) = record { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 1ab59675b6..9355478a2d 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -25,14 +25,14 @@ pub(crate) fn review_json_failure(arguments: &[String], outcome: &ArgvOutcome) - .is_some_and(|failure| failure.prog == REVIEW_PROG) } -fn text(namespace: &Namespace, name: &str) -> Option { +pub(crate) fn text(namespace: &Namespace, name: &str) -> Option { match namespace.get(name) { Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), _ => None, } } -fn flag(namespace: &Namespace, name: &str) -> bool { +pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 867ce8bb82..7f993a2320 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -41,6 +41,8 @@ pub(crate) const RUNNING_BUILD_STATUS: &str = "running"; pub(crate) const CLEAN_COMMAND: &str = "clean"; pub(crate) const INIT_COMMAND: &str = "init"; pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const SERVER_COMMAND: &str = "server"; +pub(crate) const SCHEDULER_COMMAND: &str = "scheduler"; pub(crate) const RULES_COMMAND: &str = "rules"; pub(crate) const RULES_LIST_ACTION: &str = "list"; pub(crate) const RULES_SHOW_ACTION: &str = "show"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs new file mode 100644 index 0000000000..111fc3a1a1 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -0,0 +1,135 @@ +//! Supervise native API/scheduler roles together; a failed role stops its siblings. + +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; + +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream, Value}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::write_output; + +async fn stopped(mut shutdown: watch::Receiver) { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } +} + +async fn shutdown_signal() -> Result<(), ControlError> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; + Ok(()) +} + +pub(crate) async fn run_control( + namespace: &Namespace, + standalone: bool, +) -> Result<(), ControlError> { + let address = if standalone { + None + } else { + let host: IpAddr = text(namespace, "control_host") + .unwrap_or_else(|| "127.0.0.1".into()) + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let port = match namespace.get("control_port") { + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, + _ => 8000, + }; + Some(SocketAddr::new(host, port)) + }; + let variable = text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); + let url = match std::env::var(&variable) { + Ok(value) => value, + Err(std::env::VarError::NotPresent) => { + PathBuf::from(text(namespace, "project_dir").unwrap_or_else(|| ".".into())) + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned() + } + Err(std::env::VarError::NotUnicode(_)) => { + return Err(ControlError::Invalid( + "control database variable must be UTF-8".into(), + )); + } + }; + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + let store = open_store(&url).await?; + migrate_store(&store).await?; + drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + let (stop, shutdown) = watch::channel(false); + let (_changes, changes) = watch::channel(()); + let mut tasks = tokio::task::JoinSet::new(); + if let Some(address) = address { + drop(write_output(&[( + Stream::Stderr, + "Starting control API...\n", + )])); + let listener = tokio::net::TcpListener::bind(address).await?; + drop(write_output(&[( + Stream::Stderr, + &format!("Control API ready: {}\n", listener.local_addr()?), + )])); + tasks.spawn(serve_api( + store.clone(), + listener, + stopped(shutdown.clone()), + )); + } + if standalone || !flag(namespace, "control_no_scheduler") { + drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); + let scheduler_store = store.clone(); + tasks.spawn(async move { + run_scheduler( + &scheduler_store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + changes, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Scheduler loop started; leadership is lease-guarded\n", + )])); + } + let mut result = tokio::select! { + result = tasks.join_next() => match result { + Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), + None => Ok(()), + }, + result = shutdown_signal() => result, + }; + stop.send_replace(true); + while let Some(joined) = tasks.join_next().await { + let joined = joined + .map_err(ControlError::from) + .and_then(std::convert::identity); + result = result.and(joined); + } + result +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 034b2c7bcd..c52a4a29a2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,3 +1,6 @@ pub(super) mod _cost_report_options; +pub(super) mod _run_control; pub(super) mod _run_cost; +pub(crate) mod parse_native_invocation; +pub(crate) mod run_control_invocation; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs new file mode 100644 index 0000000000..707115b4f9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -0,0 +1,49 @@ +//! Parse once with the shared command table before resolving the optional Python CLI. + +use std::ffi::OsString; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::review_json_failure; +use crate::native_commands::_helpers::terminal::{ + finish_rejected_parse, stdout_supports_color, terminal_columns, +}; +use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeParse; + +pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let settings = ArgvSettings { + columns: terminal_columns()?, + use_color, + }; + let mut host = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(_) if review_json_failure(&arguments, &outcome) => return Err(PythonNeeded::Arguments), + Err(code) => { + return Ok(NativeParse::Exit(finish_rejected_parse( + &outcome.writes, + code, + ))); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + Ok(NativeParse::Invocation { + namespace, + use_color, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs new file mode 100644 index 0000000000..699407935b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -0,0 +1,63 @@ +//! Control roles are served before Python/package discovery, including their JSON contract. + +use sqlbuild_orchestration::api::main::openapi_document::openapi_document; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::main::_run_control::run_control; + +pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { + let command = text(namespace, "command")?; + if !matches!(command.as_str(), SERVER_COMMAND | SCHEDULER_COMMAND) { + return None; + } + if flag(namespace, "control_openapi") { + return Some(match serde_json::to_string(&openapi_document()) { + Ok(document) => { + let document = document + "\n"; + let lost = write_output(&[(Stream::Stdout, document.as_str())]); + finish_lost_stdout(lost.as_ref(), 0) + } + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Control API contract generation failed\n", + )])); + 1 + } + }); + } + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control service failed to start: {error}\n"), + )])); + return Some(1); + } + }; + Some( + match runtime.block_on(run_control(namespace, command == SCHEDULER_COMMAND)) { + Ok(()) => { + drop(write_output(&[( + Stream::Stderr, + "Control service stopped successfully\n", + )])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control service failed: {error}\n"), + )])); + 1 + } + }, + ) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 40360a7de8..c92474afef 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -1,50 +1,19 @@ //! Serve one invocation without Python when its parse and project facts allow it. -use std::ffi::OsString; use std::path::Path; -use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; -use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; -use crate::native_commands::_helpers::decision::{native_invocation, review_json_failure}; +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; -use crate::native_commands::_helpers::terminal::{ - finish_rejected_parse, stdout_supports_color, terminal_columns, -}; -use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::NativeInvocation; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( - arguments: &[OsString], + namespace: &Namespace, python_cli: &Path, + use_color: bool, ) -> Result { - let arguments: Vec = arguments - .iter() - .map(|argument| argument.to_str().map(str::to_owned)) - .collect::>() - .ok_or(PythonNeeded::Arguments)?; - let use_color: bool = - !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); - let settings: ArgvSettings = ArgvSettings { - columns: terminal_columns()?, - use_color, - }; - let mut host: NativeArgvHost = NativeArgvHost::default(); - let outcome: ArgvOutcome = - parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); - if host.needs_python { - return Err(PythonNeeded::Arguments); - } - let namespace: Namespace = match outcome.parsed { - Err(_) if review_json_failure(&arguments, &outcome) => { - return Err(PythonNeeded::Arguments); - } - Err(code) => return Ok(finish_rejected_parse(&outcome.writes, code)), - Ok(namespace) if outcome.writes.is_empty() => namespace, - Ok(_) => return Err(PythonNeeded::Arguments), - }; - let invocation: NativeInvocation = native_invocation(&namespace)?; + let invocation: NativeInvocation = native_invocation(namespace)?; run_command(&invocation, python_cli, use_color) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 93db875a33..f328de4e67 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -2,6 +2,18 @@ use std::path::PathBuf; +use crate::argv::models::Namespace; + +/// Parsing can finish an invocation before package discovery or any Python handoff. +#[derive(Debug)] +pub(crate) enum NativeParse { + Exit(i32), + Invocation { + namespace: Namespace, + use_color: bool, + }, +} + /// What `sqb skills` installs. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct SkillsOptions { diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs new file mode 100644 index 0000000000..23a9b5ba97 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -0,0 +1,6 @@ +#[path = "native_commands/helpers.rs"] +mod helpers; +#[path = "native_commands/test_control.rs"] +mod test_control; +#[path = "native_commands/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs new file mode 100644 index 0000000000..5e4ae4bfde --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -0,0 +1,108 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::test_types::TestControl; + +pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { + let binary = directory.join(if cfg!(windows) { "sqb.exe" } else { "sqb" }); + std::fs::copy(env!("CARGO_BIN_EXE_sqb"), &binary) + .expect("copy standalone binary without Python CLI"); + binary +} + +pub(crate) fn start_control(arguments: &[&str]) -> TestControl { + let directory = tempfile::tempdir().expect("isolated native control project"); + let binary = isolated_binary(directory.path()); + let stderr = directory.path().join("control.log"); + let child = Command::new(binary) + .args(arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("control stderr")) + .spawn() + .expect("start native control process"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn api_ready(control: &mut TestControl) -> String { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + assert!( + control.child.try_wait().expect("child status").is_none(), + "control process exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + let log = std::fs::read_to_string(&control.stderr).expect("read control log"); + if let Some(address) = log + .lines() + .find_map(|line| line.strip_prefix("Control API ready: ")) + { + let url = format!("http://{address}"); + if client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return url; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("native API becomes ready") +} + +pub(crate) fn leader_count(control: &TestControl) -> Option { + let connection = rusqlite::Connection::open_with_flags( + control.directory.path().join(".sqlbuild/control.sqlite3"), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .ok()?; + connection + .query_row("SELECT COUNT(*) FROM sqb_scheduler_lease", [], |row| { + row.get(0) + }) + .ok() +} + +#[cfg(unix)] +pub(crate) async fn terminate(control: &mut TestControl) { + let status = Command::new("kill") + .args(["-TERM", &control.child.id().to_string()]) + .status() + .expect("send SIGTERM"); + assert!(status.success()); + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Some(status) = control.child.try_wait().expect("child exit") { + return status; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("control roles stop promptly"); + assert!( + status.success(), + "shutdown failed: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + assert!( + std::fs::read_to_string(&control.stderr) + .expect("shutdown log") + .ends_with("Control service stopped successfully\n") + ); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs new file mode 100644 index 0000000000..de16cd7367 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -0,0 +1,100 @@ +use std::process::Command; +use std::time::Duration; + +use crate::helpers::{api_ready, isolated_binary, leader_count, start_control}; + +#[test] +fn given_a_standalone_binary_when_printing_control_help_or_openapi_then_python_and_database_access_are_not_required() + { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for arguments in [ + ["server", "--help"], + ["scheduler", "--help"], + ["server", "--openapi"], + ] { + let output = Command::new(&binary) + .args(arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty()); + if arguments[1] == "--openapi" { + let document: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); + assert!(document["paths"]["/api/runs/{run_id}/events"].is_object()); + } else { + assert!(String::from_utf8_lossy(&output.stdout).starts_with("usage: sqb ")); + } + } + assert!(!directory.path().join(".sqlbuild").exists()); + let output = Command::new(&binary) + .args(["server", "--port", "65536"]) + .current_dir(directory.path()) + .env("PATH", "") + .output() + .expect("invalid port"); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("--port must be between 0 and 65535")); + assert!( + !directory.path().join(".sqlbuild").exists(), + "invalid input cannot initialize a database" + ); +} + +#[tokio::test] +async fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() + { + for scheduler in [false, true] { + let mut arguments = vec!["server", "--port", "0"]; + if !scheduler { + arguments.push("--no-scheduler"); + } + let mut control = start_control(&arguments); + let url = api_ready(&mut control).await; + let response = reqwest::Client::new() + .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) + .send() + .await + .expect("protected native API"); + assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); + tokio::time::timeout(Duration::from_secs(5), async { + while leader_count(&control) != Some(i64::from(scheduler)) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("embedded scheduler enabled exactly when requested"); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + } +} + +#[tokio::test] +async fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() + { + let mut control = start_control(&["scheduler"]); + tokio::time::timeout(Duration::from_secs(5), async { + while leader_count(&control) != Some(1) { + assert!( + control + .child + .try_wait() + .expect("scheduler status") + .is_none() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("standalone scheduler acquires leadership"); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs new file mode 100644 index 0000000000..35842e7f61 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; +use std::process::Child; + +pub(crate) struct TestControl { + pub(crate) child: Child, + pub(crate) directory: tempfile::TempDir, + pub(crate) stderr: PathBuf, +} + +impl Drop for TestControl { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml new file mode 100644 index 0000000000..7e9e614a65 --- /dev/null +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "sqlbuild-orchestration" +version.workspace = true +description = "SQLBuild control-plane storage, snapshots, scheduling and run coordination." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +axum.workspace = true +chrono.workspace = true +chrono-tz.workspace = true +croner.workspace = true +flate2.workspace = true +getrandom.workspace = true +rusqlite.workspace = true +semver.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlbuild-recall.workspace = true +sqlx-core.workspace = true +sqlx-postgres.workspace = true +subtle.workspace = true +thiserror.workspace = true +tokio.workspace = true +utoipa.workspace = true +uuid.workspace = true + +[dev-dependencies] +proptest.workspace = true +reqwest.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs new file mode 100644 index 0000000000..2920fd6ea0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -0,0 +1,38 @@ +//! Authenticate before body buffering; scope checks remain in each owning endpoint. + +use axum::extract::{Request, State}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::middleware::Next; +use axum::response::Response; + +use crate::api::errors::ApiError; +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::store::models::ControlStore; + +pub(crate) async fn authenticate( + State(store): State, + mut request: Request, + next: Next, +) -> Result { + let unauthorized = || ApiError { + status: StatusCode::UNAUTHORIZED, + message: "valid bearer credential required".into(), + }; + let bearer = request + .headers() + .get(header::AUTHORIZATION) + .ok_or_else(unauthorized)? + .to_str() + .map_err(|_| unauthorized())? + .strip_prefix("Bearer ") + .ok_or_else(unauthorized)?; + let identity = authenticate_token(&store, bearer) + .await? + .ok_or_else(unauthorized)?; + request.extensions_mut().insert(identity); + let mut response = next.run(request).await; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(response) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs new file mode 100644 index 0000000000..41d8c461c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod authentication; +pub(crate) mod runs; +pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs new file mode 100644 index 0000000000..419ec2681a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -0,0 +1,65 @@ +//! Worker reports are bound to the authenticated run and claim, never a caller-supplied actor. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::models::{RunAuthority, RunChangeRequest}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/runs/{run_id}/events", params(("run_id" = Uuid, Path)), + request_body = RunChangeRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(request): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + if request.run_id != run_id || request.claim_id != Some(claim_id) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "report does not match the authenticated run and claim".into(), + }); + } + let applied = change_run(&store, RunAuthority::Worker { run_id, claim_id }, &request).await?; + Ok(Json(MutationResponse { applied })) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/heartbeat", params(("run_id" = Uuid, Path)), + responses((status = 200, body = HeartbeatResponse), (status = 401, description = "Invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn heartbeat( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json(HeartbeatResponse { + cancel_requested: heartbeat_run(&store, (run_id, claim_id)).await?, + })) +} + +fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { + if let CredentialScope::Run { + run_id: permitted, + claim_id, + } = identity.scope + && permitted == run_id + { + return Ok(claim_id); + } + Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this run".into(), + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs new file mode 100644 index 0000000000..198925e8fc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -0,0 +1,65 @@ +//! A location credential publishes and reads only its own location's snapshots. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + request_body = SnapshotInput, + responses((status = 200, body = SnapshotRecord), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Stale generation")), security(("bearer" = [])))] +pub(crate) async fn publish( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(input): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + if input.location_id != location_id || input.target != target { + return Err(ApiError { + status: StatusCode::BAD_REQUEST, + message: "snapshot does not match the requested location and target".into(), + }); + } + Ok(Json(publish_snapshot(&store, &input).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = CurrentSnapshot), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let current = current_snapshot(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + })?; + Ok(Json(current)) +} + +fn require_location(identity: &TokenIdentity, location_id: Uuid) -> Result<(), ApiError> { + if identity.scope != (CredentialScope::Location { location_id }) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this location".into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs new file mode 100644 index 0000000000..011ed4974a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -0,0 +1,54 @@ +//! HTTP errors never return database diagnostics, connection strings, or credentials. + +use axum::Json; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; + +use crate::api::models::ApiErrorBody; +use crate::store::errors::ControlError; + +#[derive(Debug)] +pub struct ApiError { + pub(crate) status: StatusCode, + pub(crate) message: String, +} + +impl From for ApiError { + fn from(error: ControlError) -> Self { + match error { + ControlError::Invalid(message) => Self { + status: StatusCode::BAD_REQUEST, + message, + }, + ControlError::StaleGeneration | ControlError::Fenced => Self { + status: StatusCode::CONFLICT, + message: error.to_string(), + }, + _ => Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "control-store operation failed".into(), + }, + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let mut response = ( + self.status, + Json(ApiErrorBody { + error: self.message, + }), + ) + .into_response(); + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + if self.status == StatusCode::UNAUTHORIZED { + response + .headers_mut() + .insert(header::WWW_AUTHENTICATE, HeaderValue::from_static("Bearer")); + } + response + } +} diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs new file mode 100644 index 0000000000..e72ebe13fc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -0,0 +1,2 @@ +pub mod openapi_document; +pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs new file mode 100644 index 0000000000..597993eda8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -0,0 +1,19 @@ +//! Generate the HTTP contract from the request/response types used by the actual endpoints. + +use utoipa::OpenApi; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; + +use crate::api::models::ControlApi; + +#[must_use] +pub fn openapi_document() -> utoipa::openapi::OpenApi { + let mut document = ControlApi::openapi(); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "bearer", + SecurityScheme::Http(HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build()), + ); + document +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs new file mode 100644 index 0000000000..51d3b48cf4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -0,0 +1,39 @@ +//! One native server owns migrations and all storage writes; locations and workers speak HTTP. + +use axum::extract::DefaultBodyLimit; +use axum::routing::{get, post}; +use axum::{Json, Router, middleware}; + +use crate::api::_helpers::authentication::authenticate; +use crate::api::_helpers::{runs, snapshots}; +use crate::api::main::openapi_document::openapi_document; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; + +pub async fn serve_api( + store: ControlStore, + listener: tokio::net::TcpListener, + shutdown: impl Future + Send + 'static, +) -> Result<(), ControlError> { + migrate_store(&store).await?; + let protected = Router::new() + .route( + "/api/locations/{location_id}/targets/{target}/snapshots", + get(snapshots::current).post(snapshots::publish), + ) + .route("/api/runs/{run_id}/events", post(runs::report)) + .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); + let application = protected + .route( + "/api/openapi.json", + get(|| async { Json(openapi_document()) }), + ) + .with_state(store); + axum::serve(listener, application) + .with_graceful_shutdown(shutdown) + .await?; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs new file mode 100644 index 0000000000..8fa5a583a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -0,0 +1,6 @@ +//! Python-free HTTP endpoints; locations and workers have no direct database access. + +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs new file mode 100644 index 0000000000..3c3590a1ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -0,0 +1,24 @@ +//! Typed HTTP responses shared with the generated OpenAPI contract. + +use serde::{Deserialize, Serialize}; + +use crate::api::_helpers::{runs, snapshots}; + +#[derive(Debug, utoipa::OpenApi)] +#[openapi(paths(snapshots::publish, snapshots::current, runs::report, runs::heartbeat))] +pub(crate) struct ControlApi; + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct MutationResponse { + pub applied: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct HeartbeatResponse { + pub cancel_requested: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ApiErrorBody { + pub error: String, +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs new file mode 100644 index 0000000000..7a78c89f96 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs @@ -0,0 +1,31 @@ +//! Security changes and their audit records commit together, without bearer material. + +use serde_json::Value; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn audit( + transaction: ControlTransaction, + context: &AuditContext, + (event_id, action): (Uuid, &str), + (scope, before, after): (Value, Value, Value), +) -> Result { + if context.actor.trim().is_empty() || context.granting_role.trim().is_empty() { + return Err(ControlError::Invalid( + "audit actor and granting role are required".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, request_id, ip_address, scope_json, before_json, after_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", + vec![Parameter::Id(event_id), Parameter::Time(now), Parameter::Text(context.actor.clone()), + Parameter::Text(action.into()), Parameter::Text(context.granting_role.clone()), Parameter::Id(context.request_id), + Parameter::OptionalText(context.ip_address.map(|value| value.to_string())), + Parameter::Json(scope), Parameter::Json(before), Parameter::Json(after)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs new file mode 100644 index 0000000000..7093af372d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod audit; +pub(crate) mod scopes; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs new file mode 100644 index 0000000000..151288c4c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs @@ -0,0 +1,40 @@ +//! Service credentials stop being valid with their live principal, location, or run claim. + +use crate::credentials::models::CredentialScope; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Bound values in principal/location/run/claim column order for the credential insert. +pub(crate) fn scope_parameters(scope: CredentialScope) -> Vec { + match scope { + CredentialScope::Principal { principal_id } => [Some(principal_id), None, None, None], + CredentialScope::Location { location_id } => [None, Some(location_id), None, None], + CredentialScope::Run { run_id, claim_id } => [None, None, Some(run_id), Some(claim_id)], + } + .into_iter() + .map(Parameter::OptionalId) + .collect() +} + +pub(crate) async fn scope_is_active( + transaction: ControlTransaction, + scope: CredentialScope, +) -> Result<(ControlTransaction, bool), ControlError> { + let (sql, parameters) = match scope { + CredentialScope::Principal { principal_id } => ( + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ), + CredentialScope::Location { location_id } => ( + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ), + CredentialScope::Run { run_id, claim_id } => ( + "SELECT run_id FROM sqb_run_liveness WHERE run_id=$1 AND worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ), + }; + let (transaction, rows) = query(transaction, sql, parameters).await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs new file mode 100644 index 0000000000..ce3675812c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -0,0 +1,4 @@ +//! Credential secrets use 256 bits of entropy, represented as hexadecimal in bearer strings. + +pub(crate) const TOKEN_SECRET_BYTES: usize = 32; +pub(crate) const TOKEN_SECRET_HEX_LENGTH: usize = TOKEN_SECRET_BYTES * 2; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs new file mode 100644 index 0000000000..4349c0d08b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs @@ -0,0 +1,59 @@ +//! Every request sees revocation and current scope state, including across server replicas. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use subtle::ConstantTimeEq; +use uuid::Uuid; + +use crate::credentials::_helpers::scopes::scope_is_active; +use crate::credentials::constants::TOKEN_SECRET_HEX_LENGTH; +use crate::credentials::models::{StoredToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Invalid, revoked, expired and inactive-scope credentials are indistinguishable to callers. +pub async fn authenticate_token( + store: &ControlStore, + bearer: &str, +) -> Result, ControlError> { + let Some((id, secret)) = bearer.split_once('.') else { + return Ok(None); + }; + if secret.len() != TOKEN_SECRET_HEX_LENGTH + || !secret.bytes().all(|value| value.is_ascii_hexdigit()) + { + return Ok(None); + } + let Ok(id) = Uuid::parse_str(id) else { + return Ok(None); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT token_id, scope_json, secret_hash, expires_at FROM sqb_access_tokens \ + WHERE token_id=$1 AND revoked_at IS NULL AND expires_at>$2", + vec![Parameter::Id(id), Parameter::Time(now)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let token: StoredToken = serde_json::from_value(serde_json::json!(row)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex(bearer.as_bytes()); + if !bool::from(digest.as_bytes().ct_eq(token.secret_hash.as_bytes())) { + commit(transaction).await?; + return Ok(None); + } + let (transaction, active) = scope_is_active(transaction, token.scope_json).await?; + commit(transaction).await?; + Ok(active.then_some(TokenIdentity { + token_id: token.token_id, + scope: token.scope_json, + expires_at: token.expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs new file mode 100644 index 0000000000..b85b43d51e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs @@ -0,0 +1,55 @@ +//! Server-owned people and service accounts; creating one never grants a role implicitly. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::audit::audit; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn create_principal( + store: &ControlStore, + context: &AuditContext, + (name, kind): (&str, PrincipalKind), +) -> Result { + if name.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let transaction = begin(store, TransactionMode::Write).await?; + let id = Uuid::now_v7(); + let now = transaction.now; + let kind = match kind { + PrincipalKind::Person => "person", + PrincipalKind::ServiceAccount => "service_account", + }; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_principals (principal_id, name, kind, created_at) VALUES ($1,$2,$3,$4)", + vec![ + Parameter::Id(id), + Parameter::Text(name.into()), + Parameter::Text(kind.into()), + Parameter::Time(now), + ], + ) + .await?; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "principal.created"), + ( + json!({"principal_id": id}), + json!(null), + json!({"name": name, "kind": kind}), + ), + ) + .await?, + ) + .await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs new file mode 100644 index 0000000000..1ccb56aba1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -0,0 +1,78 @@ +//! Random bearer secrets are returned once; only their hashes and bounded scopes are durable. + +use std::fmt::Write; + +use chrono::{DateTime, SubsecRound, Utc}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::audit::audit; +use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; +use crate::credentials::constants::TOKEN_SECRET_BYTES; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The caller authorizes token management before entering this storage operation. +pub async fn issue_token( + store: &ControlStore, + context: &AuditContext, + scope: CredentialScope, + expires_at: DateTime, +) -> Result { + let expires_at = expires_at.trunc_subsecs(6); + let transaction = begin(store, TransactionMode::Write).await?; + if expires_at <= transaction.now { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + let (transaction, active) = scope_is_active(transaction, scope).await?; + if !active { + return Err(ControlError::Invalid("token scope is not active".into())); + } + let token_id = Uuid::now_v7(); + let mut random = [0_u8; TOKEN_SECRET_BYTES]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a secure token".into()))?; + let mut bearer = format!("{token_id}."); + for byte in random { + write!(bearer, "{byte:02x}") + .map_err(|_| ControlError::Invalid("could not format a secure token".into()))?; + } + let now = transaction.now; + let mut parameters: Vec = vec![ + Parameter::Id(token_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + ]; + parameters.extend(scope_parameters(scope)); + parameters.extend([ + Parameter::Json(json!(scope)), + Parameter::Time(now), + Parameter::Time(expires_at), + ]); + let (transaction, _) = query(transaction, + "INSERT INTO sqb_access_tokens (token_id, secret_hash, principal_id, location_id, run_id, claim_id, scope_json, created_at, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + parameters).await?; + let identity = TokenIdentity { + token_id, + scope, + expires_at, + }; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"issued"), "token.issued"), + (json!(scope), json!(null), json!(identity)), + ) + .await?, + ) + .await?; + Ok(IssuedToken { identity, bearer }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs new file mode 100644 index 0000000000..6b63069d7b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -0,0 +1,4 @@ +pub mod authenticate_token; +pub mod create_principal; +pub mod issue_token; +pub mod revoke_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs new file mode 100644 index 0000000000..6be1bb1e78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -0,0 +1,48 @@ +//! Revocation and its security audit are one transaction, without a per-process token cache. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::audit::audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Repeating a revocation does not create a second audit entry or rewrite its actor. +pub async fn revoke_token( + store: &ControlStore, + context: &AuditContext, + token_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_access_tokens SET revoked_at=$1 WHERE token_id=$2 AND revoked_at IS NULL RETURNING scope_json", + vec![Parameter::Time(now), Parameter::Id(token_id)]).await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(false); + }; + let scope = row + .get("scope_json") + .cloned() + .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), + ( + scope, + json!({"token_id": token_id, "revoked": false}), + json!({"token_id": token_id, "revoked": true}), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/mod.rs b/crates/sqlbuild-orchestration/src/credentials/mod.rs new file mode 100644 index 0000000000..a3ce0d1f39 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/mod.rs @@ -0,0 +1,6 @@ +//! Hashed, revocable credentials scoped to one principal, location, or claimed run. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs new file mode 100644 index 0000000000..f8f7d956f1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -0,0 +1,63 @@ +//! Credentials identify a caller; principal permissions are a separate authorization decision. + +use std::net::IpAddr; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrincipalKind { + Person, + ServiceAccount, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CredentialScope { + Principal { principal_id: Uuid }, + Location { location_id: Uuid }, + Run { run_id: Uuid, claim_id: Uuid }, +} + +/// Server-derived request metadata, never copied from an action's JSON body. +#[derive(Clone, Debug)] +pub struct AuditContext { + pub actor: String, + pub granting_role: String, + pub request_id: Uuid, + pub ip_address: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenIdentity { + pub token_id: Uuid, + pub scope: CredentialScope, + pub expires_at: DateTime, +} + +/// The bearer is returned once. Debug output must never contain it. +#[derive(Serialize)] +pub struct IssuedToken { + pub identity: TokenIdentity, + pub bearer: String, +} + +impl std::fmt::Debug for IssuedToken { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IssuedToken") + .field("identity", &self.identity) + .field("bearer", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredToken { + pub(crate) token_id: Uuid, + pub(crate) scope_json: CredentialScope, + pub(crate) secret_hash: String, + pub(crate) expires_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs new file mode 100644 index 0000000000..e7d2165cf9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -0,0 +1,87 @@ +//! Apply an operator fact and its projection in one transaction, with retry-safe request identity. + +use chrono::TimeDelta; + +use crate::declarations::models::{ + EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, +}; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Authorization belongs to the API boundary; `actor` is the authenticated principal, not client input. +pub async fn change_trigger( + store: &ControlStore, + request: &TriggerChangeRequest, +) -> Result { + if request.actor.trim().is_empty() { + return Err(ControlError::Invalid( + "trigger changes require an actor".into(), + )); + } + let body = + serde_json::to_value(request).map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_trigger_state_events WHERE event_id=$1", + vec![Parameter::Id(request.event_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&body) { + return Err(ControlError::Invalid( + "trigger change event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query( + transaction, + "SELECT t.definition_json, t.snapshot_id, s.started_at, s.last_tick_for, s.status_override \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.trigger_id=$1 AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)], + ) + .await?; + let mut state: EditableTriggerState = + decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + match request.change { + TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), + TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), + TriggerChange::ResetDefault => state.status_override = None, + TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), + TriggerChange::ResetCursor => state.last_tick_for = None, + } + let status = state + .status_override + .unwrap_or(state.definition_json.default_status); + let after = state.last_tick_for.unwrap_or( + state + .started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&state.definition_json.schedule, after)? + } else { + None + }; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", + vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", + vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), + Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs new file mode 100644 index 0000000000..10d6d3f28e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs @@ -0,0 +1,28 @@ +//! Read immutable operator changes in committed order. + +use uuid::Uuid; + +use crate::declarations::models::TriggerChangeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_trigger_events( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT request_json, occurred_at, snapshot_id FROM sqb_trigger_state_events WHERE trigger_id=$1 ORDER BY occurred_at, event_id LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let events = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(events) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs new file mode 100644 index 0000000000..c3ead93c9c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs @@ -0,0 +1,30 @@ +//! Current declarations and runtime state are read in one consistent transaction. + +use uuid::Uuid; + +use crate::declarations::models::TriggerRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_triggers( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.name, t.job_key, t.definition_json, s.status, s.last_tick_for, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.location_id=$1 AND t.target=$2 AND t.removed_in_snapshot_id IS NULL ORDER BY t.name", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let triggers = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(triggers) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs new file mode 100644 index 0000000000..a9078f70e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -0,0 +1,5 @@ +pub mod change_trigger; +pub mod list_trigger_events; +pub mod list_triggers; +pub(crate) mod publish_definitions; +pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs new file mode 100644 index 0000000000..c65b13468f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -0,0 +1,94 @@ +//! Publish definition projections inside the current-snapshot transaction. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::declarations::models::{Definitions, PreviousTriggerState, TriggerStatus}; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_definitions( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: Definitions, +) -> Result { + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + } + for (name, job) in definitions.jobs { + let definition = + serde_json::to_value(&job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,'job',$5,$6)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into())]).await?; + for node in job.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node)]).await?; + } + } + (transaction, _) = query(transaction, + "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + for (name, trigger) in definitions.triggers { + let id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let definition = serde_json::to_value(&trigger) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ($1,$2,$3,'schedule',$4,$5,$6,$7) \ + ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", + vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), + Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; + let previous; + (transaction, previous) = query(transaction, "SELECT started_at, last_tick_for, status_override FROM sqb_trigger_state WHERE trigger_id=$1", vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + let started_at = state.as_ref().map_or( + trigger.starts_at.unwrap_or(snapshot.published_at), + |state| state.started_at, + ); + let default = trigger.default_status; + let status = state + .as_ref() + .and_then(|state| state.status_override) + .unwrap_or(default); + let after = state + .as_ref() + .and_then(|state| state.last_tick_for) + .unwrap_or( + started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&trigger.schedule, after)? + } else { + None + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", + vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs new file mode 100644 index 0000000000..96987a439c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -0,0 +1,79 @@ +//! Validate resolved declarations before acquiring the snapshot publication lock. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; + +use crate::declarations::models::{Definitions, Schedule}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::models::CatchUp; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +fn document( + input: &SnapshotInput, + name: &str, +) -> Result, ControlError> { + input.documents.get(name).map_or_else( + || Ok(BTreeMap::new()), + |value| { + serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) + }, + ) +} + +/// Names, references and resolved selections are checked again at the native trust boundary. +pub fn snapshot_definitions(input: &SnapshotInput) -> Result { + let definitions = Definitions { + jobs: document(input, "jobs")?, + triggers: document(input, "triggers")?, + }; + for (name, job) in &definitions.jobs { + if name.trim().is_empty() + || job.description.trim().is_empty() + || job.selection.is_empty() + || job + .selection + .iter() + .any(|key| !input.nodes.contains_key(key)) + { + return Err(ControlError::Invalid(format!( + "job {name} needs a description and a nonempty selection of snapshot nodes" + ))); + } + } + for (name, trigger) in &definitions.triggers { + if name.trim().is_empty() + || trigger.description.trim().is_empty() + || !definitions.jobs.contains_key(&trigger.job_key) + { + return Err(ControlError::Invalid(format!( + "trigger {name} needs a description and a known job" + ))); + } + if let CatchUp::All { max_runs } = trigger.catch_up + && max_runs.get() > MAX_TICKS_PER_BATCH + { + return Err(ControlError::Invalid(format!( + "trigger {name} catch-up exceeds {MAX_TICKS_PER_BATCH}" + ))); + } + match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => { + parse_cron(expression, timezone)?; + } + Schedule::Calendar { times, .. } if times.len() > MAX_TICKS_PER_BATCH => { + return Err(ControlError::Invalid(format!( + "calendar {name} exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + Schedule::Calendar { .. } => {} + } + } + Ok(definitions) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs new file mode 100644 index 0000000000..5355003ff2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -0,0 +1,2 @@ +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs new file mode 100644 index 0000000000..9561c50ca1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -0,0 +1,139 @@ +//! Resolved, serializable declarations; Python objects never enter scheduler state. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::scheduler::models::{CatchUp, Overlap}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OnFailure { + #[default] + SkipDependents, + SkipDownstreamJobs, + StopGroup, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TriggerStatus { + #[default] + Running, + Paused, +} + +impl TriggerStatus { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Running => "running", + Self::Paused => "paused", + } + } +} + +/// Selections have already been resolved by the code location against the compiled snapshot. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobDefinition { + pub description: String, + pub selection: BTreeSet, + #[serde(default)] + pub settings: BTreeMap, + #[serde(default)] + pub overlap: Overlap, + #[serde(default)] + pub on_failure: OnFailure, + #[serde(default)] + pub concurrency_groups: BTreeSet, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Schedule { + Cron { + expression: String, + timezone: String, + }, + Calendar { + times: BTreeSet>, + refresh_at: DateTime, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TriggerDefinition { + pub description: String, + pub job_key: String, + pub schedule: Schedule, + #[serde(default)] + pub starts_at: Option>, + #[serde(default)] + pub default_status: TriggerStatus, + #[serde(default)] + pub catch_up: CatchUp, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Definitions { + pub jobs: BTreeMap, + pub triggers: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct TriggerRecord { + pub trigger_id: uuid::Uuid, + pub name: String, + pub job_key: String, + pub definition_json: TriggerDefinition, + pub status: TriggerStatus, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +/// Operator changes are immutable facts; clearing an override restores the published code default. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum TriggerChange { + Enable, + Pause, + ResetDefault, + SetCursor { time: DateTime }, + ResetCursor, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TriggerChangeRequest { + pub event_id: uuid::Uuid, + pub trigger_id: uuid::Uuid, + pub actor: String, + pub reason: String, + pub change: TriggerChange, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TriggerChangeRecord { + pub request_json: TriggerChangeRequest, + pub occurred_at: DateTime, + pub snapshot_id: uuid::Uuid, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct EditableTriggerState { + pub(crate) definition_json: TriggerDefinition, + pub(crate) snapshot_id: uuid::Uuid, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct PreviousTriggerState { + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs new file mode 100644 index 0000000000..d1bc86c3ef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -0,0 +1,11 @@ +//! The Python-free SQLBuild control plane. + +#![forbid(unsafe_code)] + +pub mod api; +pub mod credentials; +pub mod declarations; +pub mod runs; +pub mod scheduler; +pub mod snapshots; +pub mod store; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs new file mode 100644 index 0000000000..ba4b35be6e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs @@ -0,0 +1,120 @@ +//! Validate lifecycle authority before publishing any fact or releasing physical write claims. + +use crate::runs::models::{ + ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; + +pub(crate) fn validate_authority( + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + let allowed = match authority { + RunAuthority::Scheduler(_) => !matches!( + request.change, + RunChange::CancelRequested { .. } + | RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + ), + RunAuthority::Worker { run_id, claim_id } => { + run_id == request.run_id + && request.claim_id == Some(claim_id) + && matches!( + request.change, + RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::Recovered + | RunChange::Finished { + outcome: RunOutcome::Success + | RunOutcome::Warn + | RunOutcome::Partial + | RunOutcome::Failed + | RunOutcome::Cancelled, + .. + } + ) + } + RunAuthority::Operator(actor) => { + matches!(&request.change, RunChange::CancelRequested { actor: recorded } if !actor.trim().is_empty() && actor == recorded) + } + }; + if !allowed { + return Err(ControlError::Invalid( + "authority cannot publish this run change".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_change( + state: &ChangingRun, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + use RunStatus::{Launched, Queued, Running, Unresponsive}; + if !matches!(state.status, Queued | Launched | Running | Unresponsive) { + return Err(ControlError::Invalid("terminal runs cannot change".into())); + } + if !matches!(request.change, RunChange::CancelRequested { .. }) + && request.claim_id != state.worker_instance + { + return Err(ControlError::Invalid( + "run change does not belong to the current claim".into(), + )); + } + let valid = match request.change { + RunChange::Launched { .. } => state.worker_instance.is_some(), + RunChange::Started => { + state.worker_instance.is_some() && matches!(state.status, Queued | Launched) + } + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + matches!(state.status, Running | Unresponsive) + } + RunChange::Unresponsive => state.status == Running, + RunChange::Recovered => state.status == Unresponsive, + RunChange::CancelRequested { .. } => state.cancel_requested_at.is_none(), + RunChange::Finished { outcome, .. } => { + state.worker_instance.is_some() + || matches!(outcome, RunOutcome::Failed | RunOutcome::Cancelled) + } + }; + if !valid { + return Err(ControlError::Invalid( + "run change is inconsistent with the current lifecycle".into(), + )); + } + if let RunChange::Finished { + outcome: RunOutcome::Success | RunOutcome::Warn | RunOutcome::Partial, + .. + } = request.change + { + let count = |status: &str| { + state + .node_counts_json + .get(status) + .copied() + .unwrap_or_default() + }; + let good = count("success") + count("warn"); + let bad = count("failed") + count("skipped") + count("cancelled"); + let expected = if bad > 0 && good > 0 { + RunOutcome::Partial + } else if bad > 0 { + RunOutcome::Failed + } else if count("warn") > 0 { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + if count("pending") > 0 + || count("running") > 0 + || !matches!(request.change, RunChange::Finished { outcome, .. } if outcome == expected) + { + return Err(ControlError::Invalid( + "run outcome does not match its completed node facts".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs new file mode 100644 index 0000000000..2b65b19559 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs @@ -0,0 +1,84 @@ +//! A rejected candidate rolls back its entire savepoint, including any partial write set. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use crate::runs::models::RunRequest; +use crate::scheduler::models::Overlap; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn reserve_claims( + mut transaction: ControlTransaction, + request: &RunRequest, + limits: &BTreeMap, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.overlap != Overlap::Allow + && let Some(job_key) = &request.job_key + { + let rows; + (transaction, rows) = query( + transaction, + "SELECT r.run_id FROM sqb_runs r JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 LIMIT 1", + vec![ + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::Text(job_key.clone()), + ], + ) + .await?; + if !rows.is_empty() { + return Ok((transaction, false)); + } + } + let now = transaction.now; + for node in &request.write_set { + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_write_claims (target, node_key, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (target, node_key) DO NOTHING RETURNING node_key", + vec![Parameter::Text(request.target.clone()), Parameter::Text(node.clone()), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + for group in &request.concurrency_groups { + let limit = limits + .get(group) + .ok_or_else(|| { + ControlError::Invalid(format!("concurrency group {group} has no configured limit")) + })? + .get(); + let slots; + (transaction, slots) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 ORDER BY slot", + vec![Parameter::Text(group.clone())], + ) + .await?; + let occupied: BTreeSet = slots + .iter() + .filter_map(|row| row.get("slot").and_then(serde_json::Value::as_i64)) + .collect(); + // Lowering a limit also counts occupied slots above the new maximum. + if occupied.len() >= limit as usize { + return Ok((transaction, false)); + } + let slot = (1..=i64::from(limit)) + .find(|slot| !occupied.contains(slot)) + .ok_or_else(|| { + ControlError::Invalid("concurrency slot accounting is inconsistent".into()) + })?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_concurrency_slots (group_key, slot, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (group_key, slot) DO NOTHING RETURNING slot", + vec![Parameter::Text(group.clone()), Parameter::Integer(slot), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs new file mode 100644 index 0000000000..7ef23c279b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs @@ -0,0 +1,63 @@ +//! All orchestration transitions share Recall's invocation sequence and transaction. + +use chrono::SecondsFormat; +use serde_json::{Map, Value, json}; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::RowEnvelope; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn append_run_event( + transaction: ControlTransaction, + (run_id, event_id): (Uuid, Uuid), + kind: &str, + payload: Value, +) -> Result<(ControlTransaction, i64), ControlError> { + let invocation_id = format!("scheduler/{run_id}"); + let timestamp = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); + let (transaction, rows) = query(transaction, + "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ + WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(invocation_id.clone())]).await?; + let previous = rows.first(); + let sequence = previous + .and_then(|row| row.get("last_sequence")) + .and_then(Value::as_i64) + .map_or(Some(0), |value| value.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; + let envelope = RowEnvelope { + invocation_id, + invocation_started_at: previous + .and_then(|row| row.get("started_at")) + .and_then(Value::as_str) + .unwrap_or(×tamp) + .into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + }; + let event = Map::from_iter([ + ("event_id".into(), json!(event_id)), + ("event_type".into(), json!(kind)), + ("schema_version".into(), json!(1)), + ("occurred_at".into(), json!(timestamp)), + ("producer".into(), json!("scheduler")), + ("invocation_sequence".into(), json!(sequence)), + ("run_id".into(), json!(run_id)), + ("payload".into(), payload), + ]); + let row = event_object_row(&envelope, &event) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, outcome) = append_recall(transaction, &row).await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "run event conflicts with a durable fact".into(), + )); + } + let cursor = outcome + .storage_id + .ok_or_else(|| ControlError::Invalid("run event has no storage cursor".into()))?; + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs new file mode 100644 index 0000000000..5994d61750 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod changes; +pub(crate) mod claims; +pub(crate) mod events; +pub(crate) mod nodes; +pub(crate) mod projection; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs new file mode 100644 index 0000000000..cb8d430a89 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs @@ -0,0 +1,83 @@ +//! Updating one node adjusts bounded status counters instead of rescanning every node per event. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_node( + transaction: ControlTransaction, + request: &RunChangeRequest, + counts: &BTreeMap, +) -> Result { + let (node, status, rows, error, skipped) = match &request.change { + RunChange::NodeStarted { node_key } => (node_key, "running", None, None, None), + RunChange::NodeFinished { + node_key, + outcome, + rows_affected, + error_code, + skipped_because_of, + } => ( + node_key, + outcome.as_str(), + *rows_affected, + error_code.clone(), + skipped_because_of.clone(), + ), + _ => { + return Err(ControlError::Invalid( + "node projection requires a node event".into(), + )); + } + }; + let now = transaction.now; + let (transaction, nodes) = query( + transaction, + "SELECT status FROM sqb_run_nodes WHERE run_id=$1 AND node_key=$2 AND attempt=1", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone())], + ) + .await?; + let previous = nodes + .first() + .and_then(|row| row.get("status")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("node is outside the frozen run selection".into()))?; + if previous != "pending" && (previous != "running" || status == "running") { + return Err(ControlError::Invalid( + "node change is inconsistent with its current outcome".into(), + )); + } + let mut counts = counts.clone(); + let old = counts + .get_mut(previous) + .ok_or_else(|| ControlError::Invalid("node counter is missing".into()))?; + *old -= 1; + if *old < 0 { + return Err(ControlError::Invalid("node counter is negative".into())); + } + if *old == 0 { + counts.remove(previous); + } + *counts.entry(status.into()).or_default() += 1; + let (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=$1, started_at=CASE WHEN $1='running' THEN $2 ELSE started_at END, \ + ended_at=$3, rows_affected=$4, error_code=$5, skipped_because_of=$6 WHERE run_id=$7 AND node_key=$8 AND attempt=1", + vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != "running").then_some(now)), + rows.map_or(Parameter::Null(sqlbuild_recall::schema::models::ColumnType::Integer), Parameter::Integer), + Parameter::OptionalText(error), Parameter::OptionalText(skipped), Parameter::Id(request.run_id), Parameter::Text(node.clone())]).await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET node_counts_json=$1 WHERE run_id=$2", + vec![ + Parameter::Json(json!(counts)), + Parameter::Id(request.run_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs new file mode 100644 index 0000000000..9d0629b574 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs @@ -0,0 +1,115 @@ +//! Lifecycle projections and claim cleanup are part of the fact's transaction, never a later task. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::_helpers::nodes::project_node; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_change( + mut transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let now = transaction.now; + match &request.change { + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + return project_node(transaction, request, &state.node_counts_json).await; + } + RunChange::Launched { container_id } => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status=CASE WHEN status='queued' THEN 'launched' ELSE status END, launched_at=COALESCE(launched_at,$1) WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_run_liveness SET container_id=$1 WHERE run_id=$2", + vec![ + Parameter::OptionalText(container_id.clone()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::Started => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status='running', launched_at=COALESCE(launched_at,$1), started_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + } + RunChange::Unresponsive | RunChange::Recovered => { + let status = if request.change == RunChange::Unresponsive { + "unresponsive" + } else { + "running" + }; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1 WHERE run_id=$2", + vec![ + Parameter::Text(status.into()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::CancelRequested { .. } => { + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET cancel_requested_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)], + ) + .await?; + } + RunChange::Finished { + outcome, reason, .. + } => { + let cancelled = *outcome == RunOutcome::Cancelled; + (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=CASE WHEN status='running' THEN $1 ELSE $2 END, ended_at=$3 \ + WHERE run_id=$4 AND status IN ('pending','running')", + vec![Parameter::Text(if cancelled { "cancelled" } else { "failed" }.into()), + Parameter::Text(if cancelled { "cancelled" } else { "skipped" }.into()), Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + let counts; + (transaction, counts) = query(transaction, + "SELECT status, COUNT(*) AS count FROM sqb_run_nodes WHERE run_id=$1 GROUP BY status", + vec![Parameter::Id(request.run_id)]).await?; + let counts: BTreeMap = counts + .into_iter() + .filter_map(|row| { + Some(( + row.get("status")?.as_str()?.into(), + row.get("count")?.as_i64()?, + )) + }) + .collect(); + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1, status_reason=$2, ended_at=$3, node_counts_json=$5 WHERE run_id=$4", + vec![ + Parameter::Text(outcome.as_str().into()), + Parameter::Text(reason.clone()), + Parameter::Time(now), + Parameter::Id(request.run_id), + Parameter::Json(json!(counts)), + ], + ) + .await?; + for table in [ + "sqb_write_claims", + "sqb_concurrency_slots", + "sqb_run_liveness", + ] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE run_id=$1"), + vec![Parameter::Id(request.run_id)], + ) + .await?; + } + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs new file mode 100644 index 0000000000..f3884cb69f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -0,0 +1,104 @@ +//! Retry-safe facts, lifecycle projection and terminal claim release form one transaction. + +use serde_json::json; + +use crate::runs::_helpers::changes::{validate_authority, validate_change}; +use crate::runs::_helpers::events::append_run_event; +use crate::runs::_helpers::projection::project_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::scheduler::main::check_lease::check_lease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Authority comes from authenticated API state, not client-supplied actor or scope fields. +pub async fn change_run( + store: &ControlStore, + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result { + validate_authority(authority, request)?; + let mut transaction = begin(store, TransactionMode::Write).await?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + let body = json!({"change": request}); + let (transaction, previous) = query( + transaction, + "SELECT invocation_id, payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(request.event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("payload") != Some(&body) + || previous + .get("invocation_id") + .and_then(serde_json::Value::as_str) + != Some(format!("scheduler/{}", request.run_id).as_str()) + { + return Err(ControlError::Invalid( + "run change event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query( + transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, + )?; + validate_change(&state, request)?; + let (transaction, mut cursor) = append_run_event( + transaction, + (request.run_id, request.event_id), + request.change.event_type(), + body, + ) + .await?; + let mut transaction = project_change(transaction, request, &state).await?; + // No process can exist before a durable claim. Cancel queued work without leaving it inert forever. + if matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none() + { + let terminal = RunChangeRequest { + event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), + run_id: request.run_id, + claim_id: None, + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "cancelled before launch".into(), + exit_code: None, + }, + }; + (transaction, cursor) = append_run_event( + transaction, + (terminal.run_id, terminal.event_id), + terminal.change.event_type(), + json!({"change": terminal}), + ) + .await?; + transaction = project_change(transaction, &terminal, &state).await?; + } + let (mut transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], + ) + .await?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs new file mode 100644 index 0000000000..3a86ee6d75 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -0,0 +1,94 @@ +//! Claim priority-ordered work without allowing a blocked page to starve later candidates. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use serde_json::json; +use sqlbuild_recall::schema::models::ColumnType; +use uuid::Uuid; + +use crate::runs::_helpers::claims::reserve_claims; +use crate::runs::_helpers::events::append_run_event; +use crate::runs::models::{QueueCandidate, RunClaim}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlStore, Parameter, TransactionMode}; + +/// Limits come from one installation configuration, never from competing per-job declarations. +pub async fn claim_next_run( + store: &ControlStore, + lease: &SchedulerLease, + limits: &BTreeMap, +) -> Result, ControlError> { + let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { + " FOR UPDATE OF r SKIP LOCKED" + } else { + "" + }; + let mut after: Option = None; + loop { + let rows; + (transaction, rows) = query(transaction, + &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.cancel_requested_at IS NULL \ + AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ + AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id LIMIT 100{locking}"), + vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), + Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), + Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id))]).await?; + if rows.is_empty() { + commit(check_lease(transaction, lease).await?).await?; + return Ok(None); + } + for row in rows { + let candidate: QueueCandidate = decode_row(row)?; + (transaction, _) = query(transaction, "SAVEPOINT candidate_claim", vec![]).await?; + let reserved; + (transaction, reserved) = + reserve_claims(transaction, &candidate.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT candidate_claim", vec![]).await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + after = Some(candidate); + continue; + } + let claim = RunClaim { + claim_id: Uuid::new_v5(&candidate.request_json.run_id, b"run_claimed"), + request: candidate.request_json, + }; + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", + vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; + let cursor; + (transaction, cursor) = append_run_event( + transaction, + (claim.request.run_id, claim.claim_id), + "run_claimed", + json!({"claim": claim}), + ) + .await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![ + Parameter::Integer(cursor), + Parameter::Id(claim.request.run_id), + ], + ) + .await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(Some(claim)); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs new file mode 100644 index 0000000000..a5399c6cdf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs @@ -0,0 +1,71 @@ +//! Heartbeats update only liveness; recovery is a durable transition in that same transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::events::append_run_event; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The worker identity is taken from its authenticated run token. Return whether cancellation is pending. +pub async fn heartbeat_run( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_run_liveness SET last_heartbeat_at=$1 WHERE run_id=$2 AND worker_instance=$3 RETURNING run_id", + vec![Parameter::Time(now), Parameter::Id(run_id), Parameter::Id(claim_id)]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "heartbeat does not belong to an active run claim".into(), + )); + } + let (mut transaction, rows) = query( + transaction, + "SELECT status, last_event_storage_id, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(run_id)], + ) + .await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("claimed run does not exist".into()))?; + let cancelled = row + .get("cancel_requested_at") + .is_some_and(|value| !value.is_null()); + if row.get("status").and_then(serde_json::Value::as_str) == Some("unresponsive") { + let previous = row + .get("last_event_storage_id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| { + ControlError::Invalid("run recovery is missing its preceding event".into()) + })?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim_id, format!("recovery/{previous}").as_bytes()), + run_id, + claim_id: Some(claim_id), + change: RunChange::Recovered, + }; + let cursor; + (transaction, cursor) = append_run_event( + transaction, + (run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status='running', last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(run_id)], + ) + .await?; + } + commit(transaction).await?; + Ok(cancelled) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs new file mode 100644 index 0000000000..f0312d6f5c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs @@ -0,0 +1,29 @@ +//! Node outcomes are scoped to one frozen run, independently of current code snapshots. + +use uuid::Uuid; + +use crate::runs::models::RunNodeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The API authorizes visibility of the run before reading its nodes. +pub async fn list_run_nodes( + store: &ControlStore, + run_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT node_key, attempt, status, started_at, ended_at, rows_affected, error_code, skipped_because_of \ + FROM sqb_run_nodes WHERE run_id=$1 ORDER BY node_key, attempt", + vec![Parameter::Id(run_id)]).await?; + let nodes = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(nodes) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs new file mode 100644 index 0000000000..516302af18 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs @@ -0,0 +1,30 @@ +//! Installation API reads never mix locations or targets. + +use uuid::Uuid; + +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Return recent projected runs; per-run events remain in Recall. +pub async fn list_runs( + store: &ControlStore, + (location_id, target): (Uuid, &str), + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id, location_id, target, snapshot_id, job_key, status, queued_at, launched_at, started_at, ended_at, selection_json, node_counts_json, last_event_storage_id \ + FROM sqb_runs WHERE location_id=$1 AND target=$2 ORDER BY queued_at DESC, run_id DESC LIMIT $3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let runs = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(runs) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs new file mode 100644 index 0000000000..e3e4c9a4d4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -0,0 +1,7 @@ +pub mod change_run; +pub mod claim_next_run; +pub mod heartbeat_run; +pub mod list_run_nodes; +pub mod list_runs; +pub(crate) mod queue_run; +pub mod request_run; diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs new file mode 100644 index 0000000000..08a7101e43 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -0,0 +1,57 @@ +//! Publish request facts, queued facts and initial node projections in the caller's transaction. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::runs::_helpers::events::append_run_event; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_run( + mut transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let now = transaction.now; + let mut cursor = None; + for kind in ["run_requested", "run_queued"] { + let storage_id; + (transaction, storage_id) = append_run_event( + transaction, + ( + request.run_id, + Uuid::new_v5(&request.run_id, kind.as_bytes()), + ), + kind, + json!({"request": request}), + ) + .await?; + cursor = Some(storage_id); + } + let cursor = + cursor.ok_or_else(|| ControlError::Invalid("queued event has no storage cursor".into()))?; + let selection = serde_json::to_value(&request.selection) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex( + &serde_json::to_vec(&request.write_set) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ + requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21)", + vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), + Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), + Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), + Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), + Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), + Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; + for node in &request.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_nodes (run_id, node_key, attempt, status, writes) VALUES ($1,$2,1,'pending',$3)", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone()), Parameter::Boolean(request.write_set.contains(node))]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs new file mode 100644 index 0000000000..eb55625e5d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -0,0 +1,67 @@ +//! Queue a frozen, already authorized request without recreating work on transport retries. + +use std::collections::BTreeSet; + +use serde_json::json; + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The API must compute and authorize this selection; callers cannot supply their own write scope. +pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result { + if request.requested_by.trim().is_empty() + || request.selection.is_empty() + || !request.write_set.is_subset(&request.selection) + { + return Err(ControlError::Invalid( + "run needs an actor, selection and a contained write set".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&json!(request)) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, nodes) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ + (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3)", + vec![ + Parameter::Id(request.snapshot_id), + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + ], + ) + .await?; + let keys: BTreeSet<&str> = nodes + .iter() + .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) + .collect(); + if request + .selection + .iter() + .any(|key| !keys.contains(key.as_str())) + { + return Err(ControlError::Invalid( + "run selection is not in the pinned location and target snapshot".into(), + )); + } + commit(queue_run(transaction, request).await?).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs new file mode 100644 index 0000000000..270381a7f4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -0,0 +1,250 @@ +//! Frozen requests and rebuildable run projections; lifecycle authority remains in Recall. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunKind { + Job, + Group, + Adhoc, +} + +impl RunKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Job => "job", + Self::Group => "group", + Self::Adhoc => "adhoc", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunOrigin { + Manual, + Schedule, + Sensor, + Rerun, + Backfill, +} + +impl RunOrigin { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Schedule => "schedule", + Self::Sensor => "sensor", + Self::Rerun => "rerun", + Self::Backfill => "backfill", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + Queued, + Launched, + Running, + Unresponsive, + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +/// The authorized selection and pinned execution inputs are immutable once requested. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub image_id: Option, + pub kind: RunKind, + pub origin: RunOrigin, + pub job_key: Option, + pub parent_run_id: Option, + pub root_run_id: Uuid, + pub trigger_id: Option, + pub tick_id: Option, + pub rerun_of_run_id: Option, + pub requested_by: String, + pub priority: i64, + pub selection: BTreeSet, + pub write_set: BTreeSet, + pub settings: BTreeMap, + pub concurrency_groups: BTreeSet, + pub overlap: crate::scheduler::models::Overlap, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunRecord { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub job_key: Option, + pub status: RunStatus, + pub queued_at: DateTime, + pub launched_at: Option>, + pub started_at: Option>, + pub ended_at: Option>, + pub selection_json: BTreeSet, + pub node_counts_json: BTreeMap, + pub last_event_storage_id: i64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NodeOutcome { + Success, + Warn, + Failed, + Skipped, + Cancelled, +} + +impl NodeOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Failed => "failed", + Self::Skipped => "skipped", + Self::Cancelled => "cancelled", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunNodeRecord { + pub node_key: String, + pub attempt: i64, + pub status: String, + pub started_at: Option>, + pub ended_at: Option>, + pub rows_affected: Option, + pub error_code: Option, + pub skipped_because_of: Option, +} + +/// The database owns the claim before a launcher sees the pinned request. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunClaim { + pub claim_id: Uuid, + pub request: RunRequest, +} + +/// Terminal outcomes cannot be confused with a request to change an active status. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunOutcome { + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Partial => "partial", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + Self::PresumedFailed => "presumed_failed", + } + } +} + +/// This is a server-owned instruction, not an arbitrary status supplied by a worker. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunChange { + Launched { + container_id: Option, + }, + Started, + NodeStarted { + node_key: String, + }, + NodeFinished { + node_key: String, + outcome: NodeOutcome, + rows_affected: Option, + error_code: Option, + skipped_because_of: Option, + }, + Unresponsive, + Recovered, + CancelRequested { + actor: String, + }, + Finished { + outcome: RunOutcome, + reason: String, + exit_code: Option, + }, +} + +impl RunChange { + pub(crate) const fn event_type(&self) -> &'static str { + match self { + Self::Launched { .. } => "run_launched", + Self::Started => "run_started", + Self::NodeStarted { .. } => "run_node_started", + Self::NodeFinished { .. } => "run_node_finished", + Self::Unresponsive => "run_unresponsive", + Self::Recovered => "run_recovered", + Self::CancelRequested { .. } => "run_cancel_requested", + Self::Finished { .. } => "run_terminated", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RunChangeRequest { + pub event_id: Uuid, + pub run_id: Uuid, + pub claim_id: Option, + pub change: RunChange, +} + +/// Constructed after authentication; this authority is never deserialized from a request body. +#[derive(Clone, Copy, Debug)] +pub enum RunAuthority<'a> { + Scheduler(&'a crate::scheduler::models::SchedulerLease), + Worker { run_id: Uuid, claim_id: Uuid }, + Operator(&'a str), +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct QueueCandidate { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct ChangingRun { + pub(crate) status: RunStatus, + pub(crate) worker_instance: Option, + pub(crate) cancel_requested_at: Option>, + pub(crate) node_counts_json: BTreeMap, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs new file mode 100644 index 0000000000..7a27408a66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs @@ -0,0 +1,31 @@ +//! Map local cron times without using the cron dependency's different fold policy. + +use chrono::{DateTime, LocalResult, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::{GapInfo, Tz}; + +use crate::store::errors::ControlError; + +/// During the first fold, smaller wall times in the second fold may still be in the future. +pub(crate) fn search_start(timezone: Tz, after: DateTime) -> NaiveDateTime { + let local = after.with_timezone(&timezone).naive_local(); + match timezone.from_local_datetime(&local) { + LocalResult::Ambiguous(earlier, later) if after < later.to_utc() => { + local - (later.to_utc() - earlier.to_utc()) + } + _ => local, + } +} + +pub(crate) fn resolve_wall_time( + timezone: Tz, + wall_time: NaiveDateTime, +) -> Result, ControlError> { + match timezone.from_local_datetime(&wall_time) { + LocalResult::Single(time) => Ok(time.to_utc()), + LocalResult::Ambiguous(_, later) => Ok(later.to_utc()), + LocalResult::None => GapInfo::new(&wall_time, &timezone) + .and_then(|gap| gap.end) + .map(|time| time.to_utc()) + .ok_or_else(|| ControlError::Invalid("cron time has no representable gap end".into())), + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs new file mode 100644 index 0000000000..a3af3bf96e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -0,0 +1,58 @@ +//! Bound cron backlog work without losing omitted times or requesting an obsolete latest run. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::{Schedule, TriggerDefinition}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::main::previous_cron_time::previous_cron_time; +use crate::scheduler::models::CatchUp; +use crate::store::errors::ControlError; + +pub(crate) fn expand_schedule( + trigger: &TriggerDefinition, + from: DateTime, + now: DateTime, +) -> Result<(Vec>, Option>), ControlError> { + let Schedule::Cron { + expression, + timezone, + } = &trigger.schedule + else { + let Schedule::Calendar { times, .. } = &trigger.schedule else { + unreachable!() + }; + return Ok(( + times.range((Excluded(from), Unbounded)).copied().collect(), + None, + )); + }; + let cron = parse_cron(expression, timezone)?; + let latest = trigger.catch_up == CatchUp::Latest; + let mut due = Vec::new(); + let mut cursor = from; + for _ in 0..(MAX_TICKS_PER_BATCH - usize::from(latest)) { + cursor = next_cron_time(&cron, cursor)?; + due.push(cursor); + if cursor > now { + return Ok((due, None)); + } + } + let continuation = next_cron_time(&cron, cursor)?; + if latest && continuation <= now { + let latest = previous_cron_time(&cron, now)?; + due.push(latest); + return Ok(( + due, + Some(if latest == continuation { + next_cron_time(&cron, latest)? + } else { + continuation + }), + )); + } + Ok((due, Some(continuation))) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs new file mode 100644 index 0000000000..85bb8cc89b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod calendar; +pub(crate) mod expansion; +pub(crate) mod persistence; +pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs new file mode 100644 index 0000000000..e7c466d77b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -0,0 +1,76 @@ +//! Deterministic tick insertion gates all run facts; retries cannot queue a second run. + +use serde_json::json; +use std::collections::BTreeSet; + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn persist_ticks( + transaction: ControlTransaction, + (trigger, lease): (&DueTrigger, &SchedulerLease), + plan: &TickPlan, +) -> Result { + let now = transaction.now; + let (mut transaction, rows) = query(transaction, + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ + WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", + vec![Parameter::Id(trigger.snapshot_id), Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; + let write_set: BTreeSet = rows + .into_iter() + .filter_map(|row| { + row.get("node_key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }) + .collect(); + for action in &plan.actions { + let (outcome, run, covered) = match action.decision { + TickDecision::Requested { run_id } => ("requested", Some(run_id), None), + TickDecision::Covered { covered_by_run_id } => { + ("covered", None, Some(covered_by_run_id)) + } + }; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, details_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", + vec![Parameter::Id(action.tick_id), Parameter::Id(trigger.trigger_id), Parameter::Time(action.scheduled_for), Parameter::Time(now), Parameter::Integer(lease.epoch), + Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered), Parameter::Json(json!({}))]).await?; + if let Some(run_id) = run + && !inserted.is_empty() + { + let request = RunRequest { + run_id, + location_id: trigger.location_id, + target: trigger.target.clone(), + snapshot_id: trigger.snapshot_id, + image_id: trigger.image_id.clone(), + kind: RunKind::Job, + origin: RunOrigin::Schedule, + job_key: Some(trigger.job_key.clone()), + parent_run_id: None, + root_run_id: run_id, + trigger_id: Some(trigger.trigger_id), + tick_id: Some(action.tick_id), + rerun_of_run_id: None, + requested_by: "system:scheduler".into(), + priority: 0, + selection: trigger.job_json.selection.clone(), + write_set: write_set.clone(), + settings: trigger.job_json.settings.clone(), + concurrency_groups: trigger.job_json.concurrency_groups.clone(), + overlap: trigger.job_json.overlap, + }; + transaction = queue_run(transaction, &request).await?; + } + } + (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET last_tick_for=$1, next_due_at=$2, updated_at=$3 WHERE trigger_id=$4", + vec![Parameter::OptionalTime(plan.last_tick_for), Parameter::OptionalTime(plan.next_due_at), Parameter::Time(now), Parameter::Id(trigger.trigger_id)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs new file mode 100644 index 0000000000..4cb2683e54 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -0,0 +1,38 @@ +//! Stable IDs and dispositions, shared by catch-up and ordinary schedule evaluation. + +use chrono::{DateTime, SecondsFormat, Utc}; +use uuid::Uuid; + +use crate::scheduler::models::{TickAction, TickDecision}; + +pub(crate) fn tick_id(trigger_id: Uuid, scheduled_for: DateTime) -> Uuid { + Uuid::new_v5( + &trigger_id, + scheduled_for + .to_rfc3339_opts(SecondsFormat::Nanos, true) + .as_bytes(), + ) +} + +pub(crate) fn requested_run(tick_id: Uuid) -> Uuid { + Uuid::new_v5(&tick_id, b"run") +} + +pub(crate) fn action( + trigger_id: Uuid, + scheduled_for: DateTime, + covering_run: Option, +) -> TickAction { + let tick_id = tick_id(trigger_id, scheduled_for); + let decision = match covering_run { + Some(covered_by_run_id) => TickDecision::Covered { covered_by_run_id }, + None => TickDecision::Requested { + run_id: requested_run(tick_id), + }, + }; + TickAction { + tick_id, + scheduled_for, + decision, + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs new file mode 100644 index 0000000000..b6d1c0ff0f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -0,0 +1,8 @@ +//! Leadership timing is shared by embedded and standalone schedulers. + +/// Renew well before another process may take over a stopped scheduler. +pub const RENEW_INTERVAL_SECONDS: u64 = 5; +/// A stopped holder cannot delay failover for more than this database-clock interval. +pub const LEASE_TTL_SECONDS: i64 = 30; +/// Bound each decision batch, including covered ticks, before opening a write transaction. +pub const MAX_TICKS_PER_BATCH: usize = 10_000; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs new file mode 100644 index 0000000000..1c1bf2ec6a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs @@ -0,0 +1,37 @@ +//! Elect one scheduler under the control store's serialized writer lock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// An existing live lease is never stolen, including by a duplicate caller with the same holder. +pub async fn acquire_lease( + store: &ControlStore, + holder: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let expires_at = now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, rows) = query(transaction, + "INSERT INTO sqb_scheduler_lease (name, holder, epoch, expires_at) VALUES ('scheduler',$1,1,$2) \ + ON CONFLICT (name) DO UPDATE SET holder=excluded.holder, epoch=sqb_scheduler_lease.epoch+1, expires_at=excluded.expires_at \ + WHERE sqb_scheduler_lease.expires_at<=$3 RETURNING epoch", + vec![Parameter::Id(holder), Parameter::Time(expires_at), Parameter::Time(now)]).await?; + let lease = rows + .first() + .and_then(|row| row["epoch"].as_i64()) + .map(|epoch| SchedulerLease { + holder, + epoch, + expires_at, + }); + commit(transaction).await?; + Ok(lease) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs new file mode 100644 index 0000000000..4c9d7a9247 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs @@ -0,0 +1,22 @@ +//! Shared fence validation using a fresh database-clock reading in the writer transaction. + +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn check_lease( + transaction: ControlTransaction, + lease: &SchedulerLease, +) -> Result { + let transaction = refresh_clock(transaction).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT epoch FROM sqb_scheduler_lease WHERE name='scheduler' AND holder=$1 AND epoch=$2 AND expires_at>$3", + vec![Parameter::Id(lease.holder), Parameter::Integer(lease.epoch), Parameter::Time(now)]).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs new file mode 100644 index 0000000000..0c079450a9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs @@ -0,0 +1,81 @@ +//! Fence, decide and publish using one serialized transaction and its database clock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::_helpers::expansion::expand_schedule; +use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A removed or paused trigger is inert; a lost fence rolls back every write, including Recall. +pub async fn evaluate_trigger( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, \ + state.started_at, state.last_tick_for, state.status FROM sqb_triggers t \ + JOIN sqb_trigger_state state ON state.trigger_id=t.trigger_id JOIN sqb_snapshots s ON s.snapshot_id=t.snapshot_id \ + JOIN sqb_jobs j ON j.location_id=t.location_id AND j.target=t.target AND j.job_key=t.job_key \ + WHERE t.trigger_id=$1 AND t.kind='schedule' AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + let Some(trigger) = rows + .into_iter() + .next() + .map(decode_row::) + .transpose()? + else { + return Ok(None); + }; + if trigger.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let from = trigger.last_tick_for.unwrap_or( + trigger + .started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let (due_times, continuation) = expand_schedule(&trigger.definition_json, from, now)?; + let (transaction, rows) = query(transaction, + "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND r.status='queued' AND r.cancel_requested_at IS NULL \ + AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) ORDER BY r.queued_at, r.run_id LIMIT 1", + vec![Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; + let waiting_run = rows + .first() + .and_then(|row| row.get("run_id")) + .and_then(serde_json::Value::as_str) + .map(Uuid::parse_str) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let plan = plan_ticks( + &TickState { + trigger_id, + enabled: true, + last_tick_for: trigger.last_tick_for, + due_times, + continuation, + catch_up: trigger.definition_json.catch_up, + overlap: trigger.job_json.overlap, + waiting_run, + }, + now, + )?; + let transaction = persist_ticks(transaction, (&trigger, lease), &plan).await?; + let transaction = check_lease(transaction, lease).await?; + commit(transaction).await?; + Ok(Some(plan)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs new file mode 100644 index 0000000000..0ffa73f861 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs @@ -0,0 +1,29 @@ +//! Tick history contains immutable evaluations, including covered schedule times. + +use uuid::Uuid; + +use crate::scheduler::models::TickRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_ticks( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, reason \ + FROM sqb_ticks WHERE trigger_id=$1 ORDER BY scheduled_for DESC LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let ticks = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(ticks) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs new file mode 100644 index 0000000000..0e3a7afb49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -0,0 +1,12 @@ +pub mod acquire_lease; +pub(crate) mod check_lease; +pub mod evaluate_trigger; +pub mod list_ticks; +pub mod next_cron_time; +pub(crate) mod next_schedule_time; +pub mod parse_cron; +pub mod plan_ticks; +pub mod previous_cron_time; +pub mod renew_lease; +pub mod run_scheduler; +pub mod scheduler_cycle; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs new file mode 100644 index 0000000000..8964653860 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs @@ -0,0 +1,28 @@ +//! Calculate the first distinct UTC tick after a persisted cursor. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::calendar::{resolve_wall_time, search_start}; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Ambiguities select the later fold; missing times move to the end of the actual IANA gap. +pub fn next_cron_time( + schedule: &CronSchedule, + after: DateTime, +) -> Result, ControlError> { + let mut cursor = search_start(schedule.timezone, after).and_utc(); + loop { + let candidate = schedule + .expression + .find_next_occurrence(&cursor, false) + .map_err(|error| { + ControlError::Invalid(format!("cron has no next occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time > after { + return Ok(time); + } + cursor = candidate; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs new file mode 100644 index 0000000000..c4675ccbef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs @@ -0,0 +1,28 @@ +//! Find the next due time for a resolved cron or calendar declaration. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::Schedule; +use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::store::errors::ControlError; + +pub(crate) fn next_schedule_time( + schedule: &Schedule, + after: DateTime, +) -> Result>, ControlError> { + match schedule { + Schedule::Cron { + expression, + timezone, + } => Ok(Some(next_cron_time( + &parse_cron(expression, timezone)?, + after, + )?)), + Schedule::Calendar { times, .. } => { + Ok(times.range((Excluded(after), Unbounded)).next().copied()) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs new file mode 100644 index 0000000000..50dbeee4b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs @@ -0,0 +1,24 @@ +//! Validate POSIX minute-resolution schedules and IANA time zones before publication. + +use croner::parser::{CronParser, Seconds, Year}; + +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Day-of-month and day-of-week use POSIX OR, not Quartz AND semantics. +pub fn parse_cron(expression: &str, timezone: &str) -> Result { + let expression = CronParser::builder() + .seconds(Seconds::Disallowed) + .year(Year::Disallowed) + .dom_and_dow(false) + .build() + .parse(expression) + .map_err(|error| ControlError::Invalid(format!("invalid cron expression: {error}")))?; + let timezone = timezone + .parse() + .map_err(|error| ControlError::Invalid(format!("invalid IANA time zone: {error}")))?; + Ok(CronSchedule { + expression, + timezone, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs new file mode 100644 index 0000000000..6ddee7f820 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs @@ -0,0 +1,84 @@ +//! Pure scheduling decisions: crashes and clock jumps cannot change a tick's identity. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::ticks::{action, requested_run, tick_id}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickPlan, TickState}; +use crate::store::errors::ControlError; + +/// Return facts to append and the cursor to commit atomically with them. +pub fn plan_ticks(state: &TickState, now: DateTime) -> Result { + if !state.enabled { + return Ok(TickPlan { + actions: vec![], + last_tick_for: state.last_tick_for, + next_due_at: None, + }); + } + if state.due_times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid( + "schedule expansion exceeds the bounded tick batch".into(), + )); + } + let pending: BTreeSet> = state + .due_times + .iter() + .copied() + .filter(|time| state.last_tick_for.is_none_or(|last| *time > last)) + .collect(); + let due: Vec> = pending + .iter() + .copied() + .take_while(|time| *time <= now) + .collect(); + let covering_run = (state.overlap == Overlap::Coalesce) + .then_some(state.waiting_run) + .flatten(); + let actions: Vec = match (covering_run, state.catch_up) { + (Some(run_id), _) => due + .iter() + .map(|time| action(state.trigger_id, *time, Some(run_id))) + .collect(), + (None, CatchUp::All { max_runs }) => due + .iter() + .take(max_runs.get()) + .map(|time| action(state.trigger_id, *time, None)) + .collect(), + (None, CatchUp::Latest) => { + let latest = due.last().copied(); + let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); + due.iter() + .map(|time| { + action( + state.trigger_id, + *time, + (Some(*time) != latest).then_some(run_id).flatten(), + ) + }) + .collect() + } + }; + let last_tick_for = actions + .iter() + .rev() + .find(|action| { + state + .continuation + .is_none_or(|next| action.scheduled_for < next) + }) + .map(|action| action.scheduled_for) + .or(state.last_tick_for); + let next_due_at = pending + .into_iter() + .chain(state.continuation) + .filter(|time| last_tick_for.is_none_or(|last| *time > last)) + .min(); + Ok(TickPlan { + actions, + last_tick_for, + next_due_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs new file mode 100644 index 0000000000..de74a44c70 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs @@ -0,0 +1,30 @@ +//! Locate the latest due tick without expanding an unbounded catch-up backlog. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::calendar::resolve_wall_time; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Fold and gap mapping is identical to forward expansion, including the later ambiguous instant. +pub fn previous_cron_time( + schedule: &CronSchedule, + at: DateTime, +) -> Result, ControlError> { + let mut cursor = at.with_timezone(&schedule.timezone).naive_local().and_utc(); + let mut inclusive = true; + loop { + let candidate = schedule + .expression + .find_previous_occurrence(&cursor, inclusive) + .map_err(|error| { + ControlError::Invalid(format!("cron has no previous occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time <= at { + return Ok(time); + } + cursor = candidate; + inclusive = false; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs new file mode 100644 index 0000000000..5e9811a5a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs @@ -0,0 +1,30 @@ +//! A paused or superseded scheduler cannot renew an expired fence. + +use chrono::TimeDelta; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Renewal preserves the epoch; only takeover allocates a new epoch. +pub async fn renew_lease( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let transaction = check_lease(transaction, lease).await?; + let expires_at = transaction.now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, _) = query(transaction, + "UPDATE sqb_scheduler_lease SET expires_at=$1 WHERE name='scheduler' AND holder=$2 AND epoch=$3", + vec![Parameter::Time(expires_at), Parameter::Id(lease.holder), Parameter::Integer(lease.epoch)]).await?; + commit(transaction).await?; + Ok(SchedulerLease { + expires_at, + ..lease.clone() + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs new file mode 100644 index 0000000000..4bf8917b6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -0,0 +1,26 @@ +//! Embedded and standalone roles share the same interruptible, durable scheduling loop. + +use tokio::sync::watch; + +use crate::scheduler::main::scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Notifications shorten the wait; durable database state, not receipt of a notification, drives work. +pub async fn run_scheduler( + store: &ControlStore, + mut state: SchedulerState, + mut shutdown: watch::Receiver, + mut changes: watch::Receiver<()>, +) -> Result<(), ControlError> { + while !*shutdown.borrow() { + let cycle = scheduler_cycle(store, &mut state).await?; + tokio::select! { + _ = tokio::time::sleep(cycle.wait_for) => {}, + result = changes.changed() => { if result.is_err() { break; } }, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs new file mode 100644 index 0000000000..9e6edf9a32 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs @@ -0,0 +1,75 @@ +//! A cycle renews leadership and drains a bounded set of persisted due triggers. + +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::acquire_lease::acquire_lease; +use crate::scheduler::main::evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn scheduler_cycle( + store: &ControlStore, + state: &mut SchedulerState, +) -> Result { + let renewal = match state.lease.as_ref() { + Some(lease) => renew_lease(store, lease).await.map(Some), + None => acquire_lease(store, state.holder).await, + }; + state.lease = match renewal { + Ok(lease) => lease, + Err(ControlError::Fenced) => None, + Err(error) => return Err(error), + }; + let mut cycle = SchedulerCycle { + leader: state.lease.is_some(), + evaluated: 0, + ticks: 0, + wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), + }; + let Some(mut lease) = state.lease.clone() else { + return Ok(cycle); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + for row in rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { + cycle.evaluated += 1; + cycle.ticks += plan.actions.len(); + } + lease = renew_lease(store, &lease).await?; + } + state.lease = Some(lease); + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?; + let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(next) = next { + cycle.wait_for = (next - now) + .to_std() + .unwrap_or(Duration::ZERO) + .min(cycle.wait_for); + } + commit(transaction).await?; + Ok(cycle) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs new file mode 100644 index 0000000000..d686433c8a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -0,0 +1,9 @@ +//! Fenced scheduler leadership and deterministic decisions over durable due times. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs new file mode 100644 index 0000000000..7c3dbfde37 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -0,0 +1,150 @@ +//! Values owned by the scheduler, independent of Python and runtime timers. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::{JobDefinition, TriggerDefinition, TriggerStatus}; + +/// A holder must present the exact epoch for every scheduler write. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SchedulerLease { + pub holder: Uuid, + pub epoch: i64, + pub expires_at: DateTime, +} + +/// Process identity survives cycles; the database remains authoritative for leadership. +#[derive(Clone, Debug)] +pub struct SchedulerState { + pub(crate) holder: Uuid, + pub(crate) lease: Option, +} + +impl SchedulerState { + #[must_use] + pub const fn new(holder: Uuid) -> Self { + Self { + holder, + lease: None, + } + } +} + +#[derive(Clone, Debug)] +pub struct SchedulerCycle { + pub leader: bool, + pub evaluated: usize, + pub ticks: usize, + pub wait_for: std::time::Duration, +} + +/// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy. +#[derive(Clone, Debug)] +pub struct CronSchedule { + pub(crate) expression: croner::Cron, + pub(crate) timezone: chrono_tz::Tz, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CatchUp { + #[default] + Latest, + /// Process the oldest bounded batch; later due times remain due for the next transaction. + All { max_runs: std::num::NonZeroUsize }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Overlap { + #[default] + Coalesce, + Queue, + Allow, +} + +impl Overlap { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Coalesce => "coalesce", + Self::Queue => "queue", + Self::Allow => "allow", + } + } +} + +/// Due times come from the persisted calendar or cron expansion, never elapsed sleep time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickState { + pub trigger_id: Uuid, + pub enabled: bool, + pub last_tick_for: Option>, + pub due_times: Vec>, + /// First omitted time in a bounded expansion; never advance the cursor past an unrecorded tick. + pub continuation: Option>, + pub catch_up: CatchUp, + pub overlap: Overlap, + /// Only unclaimed waiting work can cover a later tick; executing work cannot cover future data. + pub waiting_run: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum TickDecision { + Requested { run_id: Uuid }, + Covered { covered_by_run_id: Uuid }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickAction { + pub tick_id: Uuid, + pub scheduled_for: DateTime, + pub decision: TickDecision, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickPlan { + pub actions: Vec, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DueTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) job_key: String, + pub(crate) snapshot_id: Uuid, + pub(crate) image_id: Option, + pub(crate) definition_json: TriggerDefinition, + pub(crate) job_json: JobDefinition, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status: TriggerStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TickOutcome { + Requested, + Skipped, + Covered, + Failed, + Deferred, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickRecord { + pub tick_id: Uuid, + pub trigger_id: Uuid, + pub scheduled_for: DateTime, + pub evaluated_at: DateTime, + pub epoch: i64, + pub outcome: TickOutcome, + pub run_id: Option, + pub covered_by_run_id: Option, + pub reason: Option, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs new file mode 100644 index 0000000000..e0731c81e8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_backlog.rs"] +mod test_backlog; +#[path = "tests/test_cron.rs"] +mod test_cron; +#[path = "tests/test_simulation.rs"] +mod test_simulation; +#[path = "tests/test_ticks.rs"] +mod test_ticks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs new file mode 100644 index 0000000000..bfc531a1fc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -0,0 +1,129 @@ +//! Clock-independent schedule fixtures. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use proptest::prelude::any; +use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; +use proptest::{prop_assert, prop_assert_eq}; +use uuid::Uuid; + +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickPlan, TickState}; +use crate::scheduler::tests::test_types::TickTestCase; + +pub(super) fn time(second: i64) -> DateTime { + DateTime::from_timestamp(second, 0).expect("synthetic schedule time is representable") +} + +pub(super) fn state(test_case: &TickTestCase) -> TickState { + TickState { + trigger_id: Uuid::from_u128(19), + enabled: test_case.enabled, + last_tick_for: test_case.last_second.map(time), + due_times: test_case + .due_seconds + .iter() + .map(|second| time(*second)) + .collect(), + catch_up: test_case.catch_up, + overlap: test_case.overlap, + waiting_run: test_case.active.then_some(Uuid::from_u128(31)), + continuation: None, + } +} + +pub(super) fn request_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Requested { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +pub(super) fn covered_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Covered { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +fn exercise_history(due: Vec, history: Vec<(i64, bool)>, catch_up: CatchUp) -> TestCaseResult { + let expected: BTreeSet = due.iter().copied().collect(); + let mut state = TickState { + trigger_id: Uuid::from_u128(19), + enabled: true, + last_tick_for: None, + due_times: due.into_iter().map(time).collect(), + catch_up, + overlap: Overlap::Queue, + waiting_run: None, + continuation: None, + }; + let mut committed: BTreeMap = BTreeMap::new(); + for (clock, commit_succeeds) in history { + let plan = plan_ticks(&state, time(clock)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(plan) = commit_succeeds.then_some(plan) else { + continue; + }; + for action in plan.actions { + prop_assert!(action.scheduled_for.timestamp() <= clock); + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "a committed tick was emitted twice" + ); + } + state.last_tick_for = plan.last_tick_for; + } + for _ in 0..=expected.len() { + let plan = plan_ticks(&state, time(1001)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(_) = plan.actions.first() else { + break; + }; + for action in plan.actions { + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "restart repeated a committed tick" + ); + } + state.last_tick_for = plan.last_tick_for; + } + prop_assert_eq!( + committed.keys().copied().collect::>(), + expected, + "a due tick was lost or invented" + ); + prop_assert_eq!( + committed + .values() + .map(|action| action.tick_id) + .collect::>() + .len(), + committed.len() + ); + Ok(()) +} + +pub(super) fn simulate(catch_up: CatchUp) -> Result<(), String> { + let strategy = ( + proptest::collection::vec(0_i64..1000, 0..128), + proptest::collection::vec((-100_i64..1100, any::()), 0..128), + ); + let mut property_runner = TestRunner::new(Config { + cases: 512, + failure_persistence: None, + ..Config::default() + }); + property_runner + .run(&strategy, |(due, history)| { + exercise_history(due, history, catch_up) + }) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs new file mode 100644 index 0000000000..19798dce2a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs @@ -0,0 +1,107 @@ +use std::collections::BTreeSet; +use std::num::NonZeroUsize; +use std::time::{Duration, Instant}; + +use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; +use crate::scheduler::_helpers::expansion::expand_schedule; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap, TickDecision, TickState}; +use crate::scheduler::tests::helpers::time; +use crate::scheduler::tests::test_types::BacklogTestCase; +use uuid::Uuid; + +#[test] +fn given_a_backlog_larger_than_two_batches_when_drained_then_latest_is_current_and_every_tick_is_retained() + { + let test_cases = [ + BacklogTestCase { + description: "latest records covered prefixes without requesting an obsolete run", + catch_up: CatchUp::Latest, + expected_latest: 1_200_180, + expected_count: 20_003, + }, + BacklogTestCase { + description: "all drains every requested tick in bounded batches", + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(MAX_TICKS_PER_BATCH).expect("nonzero bound"), + }, + expected_latest: 1_200_180, + expected_count: 20_003, + }, + ]; + for test_case in test_cases { + let started = Instant::now(); + let trigger = TriggerDefinition { + description: "orders backlog".into(), + job_key: "orders".into(), + starts_at: Some(time(60)), + default_status: TriggerStatus::Running, + catch_up: test_case.catch_up, + schedule: Schedule::Cron { + expression: "* * * * *".into(), + timezone: "UTC".into(), + }, + }; + let mut cursor = None; + let mut recorded: BTreeSet = BTreeSet::new(); + let mut requested: BTreeSet = BTreeSet::new(); + for _ in 0..4 { + let (due_times, continuation) = expand_schedule( + &trigger, + cursor.unwrap_or(time(0)), + time(test_case.expected_latest), + ) + .expect("bounded expansion"); + let plan = plan_ticks( + &TickState { + trigger_id: Uuid::from_u128(19), + enabled: true, + last_tick_for: cursor, + due_times, + continuation, + catch_up: test_case.catch_up, + overlap: Overlap::Queue, + waiting_run: None, + }, + time(test_case.expected_latest), + ) + .expect("bounded plan"); + for action in plan.actions { + recorded.insert(action.scheduled_for.timestamp()); + let TickDecision::Requested { .. } = action.decision else { + continue; + }; + requested.insert(action.scheduled_for.timestamp()); + } + cursor = plan.last_tick_for; + } + assert_eq!( + recorded, + (1..=test_case.expected_count) + .map(|minute| minute as i64 * 60) + .collect(), + "{}", + test_case.description + ); + assert_eq!( + requested.last().copied(), + Some(test_case.expected_latest), + "{}", + test_case.description + ); + let expected_requests = + [test_case.expected_count, 1][usize::from(test_case.catch_up == CatchUp::Latest)]; + assert_eq!( + requested.len(), + expected_requests, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "bounded cron expansion exceeded five seconds: {}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs new file mode 100644 index 0000000000..d05d1e0ae1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -0,0 +1,158 @@ +use chrono::{DateTime, SecondsFormat}; + +use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::main::previous_cron_time::previous_cron_time; +use crate::scheduler::tests::test_types::{CronTestCase, PreviousCronTestCase}; + +#[test] +fn given_cron_and_iana_time_zone_when_crossing_dst_then_utc_ticks_follow_the_explicit_policy() { + let test_cases = [ + CronTestCase { + description: "spring gap ends at 03:00, not 03:30", + expression: "30 2 * * *", + timezone: "America/New_York", + after: "2026-03-08T00:00:00Z", + expected_next: "2026-03-08T07:00:00Z", + }, + CronTestCase { + description: "ambiguous daily time uses the later fold", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T00:00:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "restarting in the first fold does not skip a smaller future wall time", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "a persisted later-fold cursor does not repeat the tick", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T06:30:00Z", + expected_next: "2026-11-02T06:30:00Z", + }, + CronTestCase { + description: "wildcard schedules also select only the later fold", + expression: "* * * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:00:00Z", + }, + CronTestCase { + description: "half-hour spring gap moves to the actual gap end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-10-03T13:00:00Z", + expected_next: "2026-10-03T15:30:00Z", + }, + CronTestCase { + description: "half-hour autumn fold uses the later 01:45", + expression: "45 1 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-04-04T13:00:00Z", + expected_next: "2026-04-04T15:15:00Z", + }, + CronTestCase { + description: "an entire skipped date moves to the first instant of the next date", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + after: "2011-12-30T00:00:00Z", + expected_next: "2011-12-30T10:00:00Z", + }, + CronTestCase { + description: "POSIX day-of-month and weekday are alternatives", + expression: "0 9 13 * MON", + timezone: "UTC", + after: "2026-10-10T12:00:00Z", + expected_next: "2026-10-12T09:00:00Z", + }, + CronTestCase { + description: "UTC schedules do not introduce local-zone shifts", + expression: "17 6 * * *", + timezone: "UTC", + after: "2026-03-08T06:17:00Z", + expected_next: "2026-03-09T06:17:00Z", + }, + ]; + for test_case in test_cases { + let schedule = + parse_cron(test_case.expression, test_case.timezone).expect("valid synthetic cron"); + let after = DateTime::parse_from_rfc3339(test_case.after) + .expect("valid start timestamp") + .to_utc(); + let observed = next_cron_time(&schedule, after).expect("next cron time exists"); + assert_eq!( + observed.to_rfc3339_opts(SecondsFormat::Secs, true), + test_case.expected_next, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_latest_tick_search_when_crossing_folds_and_gaps_then_it_agrees_with_forward_tick_policy() +{ + let test_cases = [ + PreviousCronTestCase { + description: "first fold excludes every ambiguous minute", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T05:45:00Z", + expected_previous: "2026-11-01T04:59:00Z", + }, + PreviousCronTestCase { + description: "second fold includes the later ambiguous instant", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T06:15:00Z", + expected_previous: "2026-11-01T06:15:00Z", + }, + PreviousCronTestCase { + description: "daily missing time fires exactly at the gap end", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T07:00:00Z", + expected_previous: "2026-03-08T07:00:00Z", + }, + PreviousCronTestCase { + description: "before gap end uses the preceding day", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T06:59:59Z", + expected_previous: "2026-03-07T07:30:00Z", + }, + PreviousCronTestCase { + description: "half-hour gap resolves to its actual end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + at: "2026-10-03T15:30:00Z", + expected_previous: "2026-10-03T15:30:00Z", + }, + PreviousCronTestCase { + description: "skipped-date cron fires at the first representable instant", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + at: "2011-12-30T10:00:00Z", + expected_previous: "2011-12-30T10:00:00Z", + }, + ]; + for test_case in test_cases { + let cron = parse_cron(test_case.expression, test_case.timezone).expect("valid cron"); + let at = DateTime::parse_from_rfc3339(test_case.at) + .expect("search time") + .to_utc(); + let previous = previous_cron_time(&cron, at).expect("previous scheduled instant"); + assert_eq!( + previous.to_rfc3339_opts(SecondsFormat::Secs, true), + test_case.expected_previous, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs new file mode 100644 index 0000000000..60e0de2784 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs @@ -0,0 +1,32 @@ +use std::num::NonZeroUsize; + +use crate::scheduler::models::CatchUp; +use crate::scheduler::tests::helpers::simulate; +use crate::scheduler::tests::test_types::SimulationTestCase; + +#[test] +fn given_random_crashes_and_clock_jumps_when_restarting_then_every_due_tick_commits_once() { + let test_cases = [ + SimulationTestCase { + description: "latest catch-up keeps covered tick history through crashes", + catch_up: CatchUp::Latest, + expected_success: true, + }, + SimulationTestCase { + description: "bounded all-catch-up drains rather than loses the backlog", + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(3).expect("nonzero test batch"), + }, + expected_success: true, + }, + ]; + for test_case in test_cases { + let observed = simulate(test_case.catch_up); + assert_eq!( + observed.is_ok(), + test_case.expected_success, + "{}: {observed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs new file mode 100644 index 0000000000..de1eaeeba9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -0,0 +1,146 @@ +use std::num::NonZeroUsize; + +use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap}; +use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; +use crate::scheduler::tests::test_types::TickTestCase; + +#[test] +fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_ticks() { + let test_cases = [ + TickTestCase { + description: "latest catch-up covers earlier ticks and deduplicates calendar entries", + enabled: true, + due_seconds: vec![30, 10, 20, 10, 40], + last_second: None, + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![30], + expected_covered: vec![10, 20], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "all catch-up leaves the unprocessed backlog due instead of skipping it", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: None, + now_second: 40, + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(2).expect("nonzero test batch"), + }, + overlap: Overlap::Queue, + active: false, + expected_requests: vec![10, 20], + expected_covered: vec![], + expected_last: Some(20), + expected_next: Some(30), + }, + TickTestCase { + description: "coalesced ticks are covered by the already waiting run", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: true, + expected_requests: vec![], + expected_covered: vec![20, 30], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "allow does not coalesce even when a prior run is active", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(5).expect("nonzero test batch"), + }, + overlap: Overlap::Allow, + active: true, + expected_requests: vec![20, 30], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "a backward clock jump cannot move the durable cursor backward", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(30), + now_second: 20, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "pause retains the cursor and does not advertise an overdue next tick", + enabled: false, + due_seconds: vec![10, 20, 30], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(10), + expected_next: None, + }, + ]; + for test_case in test_cases { + let input = state(&test_case); + let plan = plan_ticks(&input, time(test_case.now_second)).expect("plan valid schedule"); + assert_eq!( + request_seconds(&plan), + test_case.expected_requests, + "{}", + test_case.description + ); + assert_eq!( + covered_seconds(&plan), + test_case.expected_covered, + "{}", + test_case.description + ); + assert_eq!( + plan.last_tick_for.map(|time| time.timestamp()), + test_case.expected_last + ); + assert_eq!( + plan.next_due_at.map(|time| time.timestamp()), + test_case.expected_next + ); + let retried = plan_ticks(&input, time(test_case.now_second)).expect("retry schedule"); + assert_eq!( + plan, retried, + "a retry has exactly the same deterministic tick and run IDs" + ); + let tick_ids: std::collections::BTreeSet<_> = + plan.actions.iter().map(|action| action.tick_id).collect(); + assert_eq!( + tick_ids.len(), + test_case.expected_requests.len() + test_case.expected_covered.len() + ); + let advanced = crate::scheduler::models::TickState { + last_tick_for: plan.last_tick_for, + ..input + }; + let second = + plan_ticks(&advanced, time(test_case.now_second)).expect("plan from committed cursor"); + assert!(second.actions.iter().all(|next| { + plan.actions + .iter() + .all(|before| before.tick_id != next.tick_id) + })); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs new file mode 100644 index 0000000000..ee92508cab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -0,0 +1,52 @@ +//! Inputs make cursor, catch-up, and overlap decisions observably different. + +use crate::scheduler::models::{CatchUp, Overlap}; + +#[derive(Debug)] +pub(super) struct TickTestCase { + pub(super) description: &'static str, + pub(super) enabled: bool, + pub(super) due_seconds: Vec, + pub(super) last_second: Option, + pub(super) now_second: i64, + pub(super) catch_up: CatchUp, + pub(super) overlap: Overlap, + pub(super) active: bool, + pub(super) expected_requests: Vec, + pub(super) expected_covered: Vec, + pub(super) expected_last: Option, + pub(super) expected_next: Option, +} + +#[derive(Debug)] +pub(super) struct CronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) after: &'static str, + pub(super) expected_next: &'static str, +} + +#[derive(Debug)] +pub(super) struct PreviousCronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) at: &'static str, + pub(super) expected_previous: &'static str, +} + +#[derive(Debug)] +pub(super) struct BacklogTestCase { + pub(super) description: &'static str, + pub(super) catch_up: CatchUp, + pub(super) expected_latest: i64, + pub(super) expected_count: usize, +} + +#[derive(Debug)] +pub(super) struct SimulationTestCase { + pub(super) description: &'static str, + pub(super) catch_up: CatchUp, + pub(super) expected_success: bool, +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs new file mode 100644 index 0000000000..4ff3f7b1f3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -0,0 +1,72 @@ +//! Canonical content addresses and compressed immutable bodies. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; + +use flate2::{Compression, read::GzDecoder, write::GzEncoder}; +use serde_json::Value; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::models::{PreparedNode, SnapshotNode}; +use crate::store::errors::ControlError; + +fn canonical(value: &Value) -> Value { + match value { + Value::Object(values) => { + let sorted: BTreeMap<_, _> = values + .iter() + .map(|(key, value)| (key.clone(), canonical(value))) + .collect(); + Value::Object(sorted.into_iter().collect()) + } + Value::Array(values) => Value::Array(values.iter().map(canonical).collect()), + _ => value.clone(), + } +} + +pub(crate) fn json_bytes(value: &Value) -> Result, ControlError> { + serde_json::to_vec(&canonical(value)).map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn compress(bytes: &[u8]) -> Result, ControlError> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} + +pub(crate) fn decompress(value: &Value) -> Result, ControlError> { + let bytes: Vec = serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut output: Vec = Vec::new(); + GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; + Ok(output) +} + +pub(crate) fn prepare_node(node: &SnapshotNode) -> Result { + let mut sources: BTreeMap<&'static str, (String, Vec)> = BTreeMap::new(); + for (kind, text) in [ + ("raw_sql", &node.raw_sql), + ("compiled_sql", &node.compiled_sql), + ("python_source", &node.python_source), + ] { + if let Some(text) = text { + sources.insert( + kind, + (sha256_hex(text.as_bytes()), compress(text.as_bytes())?), + ); + } + } + let source_hashes: BTreeMap<_, _> = sources + .iter() + .map(|(kind, (hash, _))| (*kind, hash)) + .collect(); + let definition = json_bytes( + &serde_json::json!({"kind": node.kind, "definition": node.definition, "sources": source_hashes}), + )?; + Ok(PreparedNode { + kind: node.kind.clone(), + hash: sha256_hex(&definition), + definition: compress(&definition)?, + sources, + }) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs new file mode 100644 index 0000000000..8853fb96b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod content; +pub(crate) mod preparation; +pub(crate) mod publication; +pub(crate) mod reading; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs new file mode 100644 index 0000000000..310f723503 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -0,0 +1,74 @@ +//! Validate and content-address a successful compile before holding a database write lock. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::_helpers::content::{compress, json_bytes, prepare_node}; +use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; +use crate::store::errors::ControlError; + +pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result { + if input.target.trim().is_empty() + || input.environment_hash.is_empty() + || input.generation <= 0 + || input + .nodes + .iter() + .any(|(key, node)| key.is_empty() || node.kind.is_empty()) + { + return Err(ControlError::Invalid( + "target, environment, positive generation, and named nodes are required".into(), + )); + } + let mut nodes: BTreeMap = BTreeMap::new(); + for (key, node) in &input.nodes { + nodes.insert(key.clone(), prepare_node(node)?); + } + let hashes: BTreeMap = nodes + .iter() + .map(|(key, node)| (key.clone(), node.hash.clone())) + .collect(); + let documents = prepare_documents(input)?; + let document_hashes: BTreeMap<_, _> = documents + .iter() + .map(|(kind, (hash, _))| (kind, hash)) + .collect(); + let manifest_digest = sha256_hex(&json_bytes(&serde_json::json!({ + "nodes": hashes, "documents": document_hashes, "environment_hash": input.environment_hash, + "image_id": input.image_id, "target": input.target, "sqb_version": input.sqb_version, + "git_revision": input.git_revision, "git_dirty": input.git_dirty, "git_author": input.git_author, + }))?); + Ok(PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + }) +} + +fn prepare_documents( + input: &SnapshotInput, +) -> Result)>, ControlError> { + let mut documents: BTreeMap)> = BTreeMap::new(); + for (kind, value) in &input.documents { + if ![ + "jobs", + "groups", + "triggers", + "roles", + "lineage", + "columns", + "providers", + ] + .contains(&kind.as_str()) + { + return Err(ControlError::Invalid(format!( + "unknown snapshot document {kind}" + ))); + } + let bytes = json_bytes(value)?; + documents.insert(kind.clone(), (sha256_hex(&bytes), compress(&bytes)?)); + } + Ok(documents) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs new file mode 100644 index 0000000000..eb7b841932 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -0,0 +1,34 @@ +//! Immutable body inserts shared by snapshot publication. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::snapshots::models::PreparedNode; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn store_nodes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + nodes: BTreeMap, +) -> Result { + for (key, node) in nodes { + (transaction, _) = query(transaction, + "INSERT INTO sqb_node_definitions (node_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(node.hash.clone()), Parameter::Bytes(node.definition)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_nodes (snapshot_id, node_key, node_kind, node_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(node.kind), Parameter::Text(node.hash)]).await?; + for (kind, (hash, body)) in node.sources { + (transaction, _) = query(transaction, + "INSERT INTO sqb_source_texts (text_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_sources (snapshot_id, node_key, kind, text_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(kind.into()), Parameter::Text(hash)]).await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs new file mode 100644 index 0000000000..91d5c5f6b0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -0,0 +1,57 @@ +//! Decode stored bodies and attach source text without evaluating it. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::snapshots::_helpers::content::decompress; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::types::Row; + +fn decode_body(value: &Value) -> Result { + serde_json::from_slice(&decompress(value)?) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn read_nodes(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "node_key") +} + +fn read_bodies( + rows: Vec, + key: &str, +) -> Result, ControlError> { + rows.into_iter() + .map(|row| { + let body: T = decode_body(&row["body"])?; + Ok((row[key].as_str().unwrap_or_default().to_owned(), body)) + }) + .collect() +} + +pub(crate) fn attach_sources( + mut nodes: BTreeMap, + rows: Vec, +) -> Result, ControlError> { + for row in rows { + let key = row["node_key"].as_str().unwrap_or_default(); + let node = nodes + .get_mut(key) + .ok_or_else(|| ControlError::Invalid("snapshot source has no node".into()))?; + let text = String::from_utf8(decompress(&row["body"])?) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + match row["kind"].as_str() { + Some("raw_sql") => node.raw_sql = Some(text), + Some("compiled_sql") => node.compiled_sql = Some(text), + Some("python_source") => node.python_source = Some(text), + _ => return Err(ControlError::Invalid("unknown stored source kind".into())), + } + } + Ok(nodes) +} + +pub(crate) fn read_documents(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "kind") +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs new file mode 100644 index 0000000000..b668008c38 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -0,0 +1,53 @@ +//! Read one consistent generation without locking out snapshot publishers. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::{attach_sources, read_documents, read_nodes}; +use crate::snapshots::models::{CurrentSnapshot, SnapshotChange, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The last successfully published generation, or none before the first successful compile. +pub async fn current_snapshot( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let Some(row) = rows.into_iter().next() else { + return Ok(None); + }; + let record: SnapshotRecord = decode_row(row)?; + let (transaction, rows) = query(transaction, + "SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1 ORDER BY n.node_key", + vec![Parameter::Id(record.snapshot_id)]).await?; + let nodes = read_nodes(rows)?; + let (transaction, rows) = query(transaction, + "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash WHERE s.snapshot_id=$1", + vec![Parameter::Id(record.snapshot_id)]).await?; + let nodes = attach_sources(nodes, rows)?; + let (transaction, rows) = query(transaction, + "SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1", + vec![Parameter::Id(record.snapshot_id)]).await?; + let documents = read_documents(rows)?; + let (transaction, rows) = query(transaction, + "SELECT change_kind, subject_key, before_hash, after_hash FROM sqb_snapshot_changes WHERE snapshot_id=$1 ORDER BY change_seq", + vec![Parameter::Id(record.snapshot_id)]).await?; + let changes: Vec = + rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + Ok(Some(CurrentSnapshot { + record, + nodes, + documents, + changes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs new file mode 100644 index 0000000000..da2b1bcca6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -0,0 +1,2 @@ +pub mod current_snapshot; +pub mod publish_snapshot; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs new file mode 100644 index 0000000000..0861207ebc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -0,0 +1,118 @@ +//! Publish a complete generation and its current pointer in one transaction. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; +use uuid::Uuid; + +use crate::declarations::main::publish_definitions::publish_definitions; +use crate::declarations::main::snapshot_definitions::snapshot_definitions; +use crate::snapshots::_helpers::preparation::prepare_snapshot; +use crate::snapshots::_helpers::publication::store_nodes; +use crate::snapshots::models::{PreparedSnapshot, SnapshotChange, SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Reject stale generations without changing the last good snapshot. +pub async fn publish_snapshot( + store: &ControlStore, + input: &SnapshotInput, +) -> Result { + let definitions = snapshot_definitions(input)?; + let PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + } = prepare_snapshot(input)?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query(transaction, + "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + if previous + .first() + .and_then(|row| row.get("generation")) + .and_then(Value::as_i64) + .is_some_and(|generation| generation >= input.generation) + { + return Err(ControlError::StaleGeneration); + } + let (mut transaction, previous) = query(transaction, + "SELECT n.node_key, n.node_hash FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + let previous: BTreeMap = previous + .into_iter() + .map(|row| { + ( + row["node_key"].as_str().unwrap_or_default().into(), + row["node_hash"].as_str().unwrap_or_default().into(), + ) + }) + .collect(); + let record = SnapshotRecord { + snapshot_id: Uuid::now_v7(), + location_id: input.location_id, + target: input.target.clone(), + generation: input.generation, + manifest_digest, + published_at: transaction.now, + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_revision, git_dirty, git_author, image_id, \ + environment_hash, sqb_version, snapshot_schema_version, manifest_digest, published_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,1,$11,$12)", vec![ + Parameter::Id(record.snapshot_id), Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Integer(input.generation), Parameter::OptionalText(input.git_revision.clone()), Parameter::Boolean(input.git_dirty), + Parameter::OptionalText(input.git_author.clone()), Parameter::OptionalText(input.image_id.clone()), + Parameter::Text(input.environment_hash.clone()), Parameter::Text(input.sqb_version.clone()), + Parameter::Text(record.manifest_digest.clone()), Parameter::Time(record.published_at), + ]).await?; + transaction = store_nodes(transaction, record.snapshot_id, nodes).await?; + for (kind, (hash, body)) in documents { + (transaction, _) = query(transaction, + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; + } + let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); + let mut sequence = 0; + for key in keys { + let before = previous.get(key); + let after = hashes.get(key); + if before == after { + continue; + } + let change = SnapshotChange { + change_kind: match (before, after) { + (None, _) => "added", + (_, None) => "removed", + _ => "changed", + } + .into(), + subject_key: key.clone(), + before_hash: before.cloned(), + after_hash: after.cloned(), + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ + VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(record.snapshot_id), Parameter::Integer(sequence), + Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), + Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; + sequence += 1; + } + (transaction, _) = query(transaction, + "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; + transaction = publish_definitions(transaction, &record, definitions).await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs new file mode 100644 index 0000000000..6f1e5da193 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -0,0 +1,79 @@ +//! Snapshot inputs are data only: no executable user objects cross into the control plane. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// A compiled node and its display-only source. SQL and Python are never run by the server. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotNode { + pub kind: String, + pub definition: Value, + pub raw_sql: Option, + pub compiled_sql: Option, + pub python_source: Option, +} + +/// A location's successful compile for one target. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotInput { + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub git_revision: Option, + pub git_dirty: bool, + pub git_author: Option, + pub image_id: Option, + pub environment_hash: String, + pub sqb_version: String, + pub nodes: BTreeMap, + pub documents: BTreeMap, +} + +/// Immutable publication metadata. Generations increase independently per location and target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotRecord { + pub snapshot_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub manifest_digest: String, + pub published_at: DateTime, +} + +/// Node-level changes do not duplicate unchanged definitions or source text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotChange { + pub change_kind: String, + pub subject_key: String, + pub before_hash: Option, + pub after_hash: Option, +} + +/// The current successful snapshot, read with a consistent generation and change set. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CurrentSnapshot { + pub record: SnapshotRecord, + pub nodes: BTreeMap, + pub documents: BTreeMap, + pub changes: Vec, +} + +#[derive(Debug)] +pub(crate) struct PreparedNode { + pub(crate) kind: String, + pub(crate) hash: String, + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, Vec)>, +} + +#[derive(Debug)] +pub(crate) struct PreparedSnapshot { + pub(crate) nodes: BTreeMap, + pub(crate) hashes: BTreeMap, + pub(crate) documents: BTreeMap)>, + pub(crate) manifest_digest: String, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/database.rs b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs new file mode 100644 index 0000000000..433dd5b495 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs @@ -0,0 +1,154 @@ +//! Typed binding and row decoding for the two control-store engines. + +use chrono::{DateTime, SecondsFormat, Utc}; +use rusqlite::types::{Value as SqliteValue, ValueRef}; +use serde_json::Value; +use sqlbuild_recall::schema::models::ColumnType; +use sqlx_core::{column::Column, row::Row as _, type_info::TypeInfo, value::ValueRef as _}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::Parameter; +use crate::store::types::Row; + +pub(crate) fn sqlite_value(parameter: &Parameter) -> SqliteValue { + match parameter { + Parameter::Null(_) => SqliteValue::Null, + Parameter::Text(value) => SqliteValue::Text(value.clone()), + Parameter::OptionalText(value) => { + value.clone().map_or(SqliteValue::Null, SqliteValue::Text) + } + Parameter::Bytes(value) => SqliteValue::Blob(value.clone()), + Parameter::Integer(value) => SqliteValue::Integer(*value), + Parameter::Real(value) => SqliteValue::Real(*value), + Parameter::Json(value) => SqliteValue::Text(value.to_string()), + Parameter::Boolean(value) => SqliteValue::Integer(i64::from(*value)), + Parameter::Id(value) => SqliteValue::Text(value.to_string()), + Parameter::OptionalId(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_string()) + }), + Parameter::Time(value) => { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + } + Parameter::OptionalTime(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + }), + } +} + +pub(crate) fn sqlite_query( + connection: &rusqlite::Connection, + sql: &str, + parameters: &[Parameter], +) -> Result, ControlError> { + let values: Vec = parameters.iter().map(sqlite_value).collect(); + let mut statement = connection.prepare(sql)?; + let columns: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let declared_types: Vec> = statement + .columns() + .into_iter() + .map(|column| column.decl_type().map(str::to_ascii_uppercase)) + .collect(); + for (index, value) in values.into_iter().enumerate() { + let name = format!("${}", index + 1); + let position = statement + .parameter_index(&name)? + .ok_or_else(|| ControlError::Invalid(format!("missing SQL parameter {name}")))?; + statement.raw_bind_parameter(position, value)?; + } + let mut query = statement.raw_query(); + let mut result: Vec = Vec::new(); + while let Some(row) = query.next()? { + let mut mapped = Row::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index)? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) if declared_types[index].as_deref() == Some("BOOLEAN") => { + Value::Bool(value != 0) + } + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => Value::from(String::from_utf8_lossy(value).into_owned()), + ValueRef::Blob(value) => Value::from(value.to_vec()), + }; + let value = if declared_types[index].as_deref() == Some("JSON") { + value.as_str().map_or_else( + || Ok(value.clone()), + |text| { + serde_json::from_str(text) + .map_err(|error| ControlError::Invalid(error.to_string())) + }, + )? + } else { + value + }; + mapped.insert(name.clone(), value); + } + result.push(mapped); + } + Ok(result) +} + +pub(crate) async fn postgres_query( + transaction: &mut sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>, + sql: &str, + parameters: Vec, +) -> Result, ControlError> { + let mut query = sqlx_core::query::query(sql); + for parameter in parameters { + query = match parameter { + Parameter::Null(kind) => match kind { + ColumnType::Text => query.bind(None::), + ColumnType::Timestamp => query.bind(None::>), + ColumnType::Json => query.bind(None::), + ColumnType::Integer => query.bind(None::), + ColumnType::Boolean => query.bind(None::), + ColumnType::Real => query.bind(None::), + ColumnType::Blob => query.bind(None::>), + }, + Parameter::Text(value) => query.bind(value), + Parameter::OptionalText(value) => query.bind(value), + Parameter::Bytes(value) => query.bind(value), + Parameter::Integer(value) => query.bind(value), + Parameter::Real(value) => query.bind(value), + Parameter::Json(value) => query.bind(value), + Parameter::Boolean(value) => query.bind(value), + Parameter::Id(value) => query.bind(value), + Parameter::OptionalId(value) => query.bind(value), + Parameter::Time(value) => query.bind(value), + Parameter::OptionalTime(value) => query.bind(value), + }; + } + let mut result: Vec = Vec::new(); + for row in query.fetch_all(&mut **transaction).await? { + let mut mapped = Row::new(); + for (index, column) in row.columns().iter().enumerate() { + let raw = row.try_get_raw(index)?; + let value = if raw.is_null() { + Value::Null + } else { + match column.type_info().name() { + "BOOL" => Value::from(row.try_get::(index)?), + "INT8" => Value::from(row.try_get::(index)?), + "INT4" => Value::from(row.try_get::(index)?), + "FLOAT8" => Value::from(row.try_get::(index)?), + "UUID" => Value::from(row.try_get::(index)?.to_string()), + "TIMESTAMPTZ" => Value::from( + row.try_get::, _>(index)? + .to_rfc3339_opts(SecondsFormat::Micros, true), + ), + "JSONB" | "JSON" => row.try_get::(index)?, + "BYTEA" => Value::from(row.try_get::, _>(index)?), + _ => Value::from(row.try_get::(index)?), + } + }; + mapped.insert(column.name().to_owned(), value); + } + result.push(mapped); + } + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs new file mode 100644 index 0000000000..53420e77dc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -0,0 +1,13 @@ +//! One schema template, rendered only at the database type boundary. + +use crate::store::models::ControlStore; + +pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); + +pub(crate) fn render(store: &ControlStore, sql: &str) -> String { + let postgres = matches!(store, ControlStore::Postgres(_)); + sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) + .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) + .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) + .replace("{json}", if postgres { "JSONB" } else { "JSON" }) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..08fcf1b81c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod database; +pub(crate) mod migration; +pub(crate) mod open_sqlite; +pub(crate) mod recall_rows; +pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs new file mode 100644 index 0000000000..9408ebaa14 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -0,0 +1,37 @@ +//! Configure a file-backed SQLite control store without interpreting filenames as URIs. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::store::errors::ControlError; + +pub(crate) fn sqlite_path(url: &str) -> Result { + let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); + if path.as_os_str().is_empty() + || path == Path::new(":memory:") + || url.contains("://") && !url.starts_with("sqlite://") + { + return Err(ControlError::Invalid( + "control-store URL must name PostgreSQL or a SQLite file".into(), + )); + } + Ok(path) +} + +pub(crate) fn open_sqlite(path: PathBuf) -> Result<(), ControlError> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let connection = rusqlite::Connection::open_with_flags( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs new file mode 100644 index 0000000000..e1827a832f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs @@ -0,0 +1,111 @@ +//! Insert native Recall rows and identify retry conflicts within a control transaction. + +use std::collections::HashSet; + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::models::{RecallRow, RecallValue}; +use sqlbuild_recall::schema::main::postgres_partition_statement::postgres_partition_statement; +use sqlbuild_recall::schema::main::table_columns::table_columns; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::{ColumnType, Table}; +use sqlbuild_recall::store::main::conflict_evidence::conflict_evidence; +use sqlbuild_recall::store::main::relational_insert::relational_insert; +use sqlbuild_recall::store::main::relational_lookup::relational_lookup; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction, Parameter, RecallAppendOutcome}; + +fn parameters(values: &[(&RecallValue, ColumnType)]) -> Result, ControlError> { + values + .iter() + .map(|(value, kind)| { + Ok(match (value, kind) { + (RecallValue::Null, kind) => Parameter::Null(*kind), + (RecallValue::Integer(value), ColumnType::Boolean) => { + Parameter::Boolean(*value != 0) + } + (RecallValue::Integer(value), _) => Parameter::Integer(*value), + (RecallValue::Real(value), _) => Parameter::Real(*value), + (RecallValue::Text(value), ColumnType::Json) => Parameter::Json( + serde_json::from_str(value) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ), + (RecallValue::Text(value), ColumnType::Timestamp) => Parameter::Time( + DateTime::parse_from_rfc3339(value) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(), + ), + (RecallValue::Text(value), _) => Parameter::Text(value.clone()), + (RecallValue::Blob(value), _) => Parameter::Bytes(value.clone()), + }) + }) + .collect() +} + +pub(crate) fn text<'a>(row: &'a RecallRow, name: &str) -> Result<&'a str, ControlError> { + match row.value(name) { + Some(RecallValue::Text(value)) => Ok(value), + _ => Err(ControlError::Invalid(format!("Recall {name} must be text"))), + } +} + +pub(crate) async fn write_row( + mut transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let postgres = matches!(transaction.connection, Some(Connection::Postgres(_))); + if postgres && row.table != Table::Artifacts { + let date = DateTime::parse_from_rfc3339(text(row, table_spec(row.table).expiry_column)?) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc() + .date_naive(); + let statement = postgres_partition_statement(row.table, date) + .ok_or_else(|| ControlError::Invalid("Recall partition date is out of range".into()))?; + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + let ingested = RecallValue::Text(transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true)); + let columns: HashSet = table_columns(row.table) + .into_iter() + .map(|column| column.name.to_owned()) + .collect(); + let insert = relational_insert((row, &columns), &ingested, postgres); + let returning = if row.table == Table::Events { + "storage_id" + } else { + "1 AS inserted" + }; + let (transaction, rows) = query( + transaction, + &format!("{} RETURNING {returning}", insert.sql), + parameters(&insert.values)?, + ) + .await?; + if let Some(found) = rows.first() { + return Ok(( + transaction, + RecallAppendOutcome { + inserted: true, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: None, + }, + )); + } + let lookup = relational_lookup(row, postgres, true); + let (transaction, rows) = query(transaction, &lookup.sql, parameters(&lookup.values)?).await?; + let found = rows.first().ok_or_else(|| { + ControlError::Invalid("Recall duplicate has no matching stored key".into()) + })?; + let existing = found + .get("content_hash") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("stored Recall row has no content hash".into()))?; + Ok(( + transaction, + RecallAppendOutcome { + inserted: false, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: conflict_evidence(row, existing), + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs new file mode 100644 index 0000000000..d63c9b5681 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs @@ -0,0 +1,13 @@ +//! Embed Recall in the control database so run facts and projections share one transaction. + +use sqlbuild_recall::schema::main::control_table_statements::control_table_statements; +use sqlbuild_recall::schema::main::control_view_statements::control_view_statements; + +use crate::store::models::ControlStore; + +pub(crate) fn recall_schema(store: &ControlStore) -> Vec { + let postgres = matches!(store, ControlStore::Postgres(_)); + let mut statements = control_table_statements(postgres); + statements.extend(control_view_statements(postgres)); + statements +} diff --git a/crates/sqlbuild-orchestration/src/store/errors.rs b/crates/sqlbuild-orchestration/src/store/errors.rs new file mode 100644 index 0000000000..54a8a6d29d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/errors.rs @@ -0,0 +1,22 @@ +//! Control-store failures; callers must not publish a successful state after one. + +/// A rejected control-plane operation or failed durable write. +#[derive(Debug, thiserror::Error)] +pub enum ControlError { + #[error("control store: {0}")] + Database(#[from] sqlx_core::Error), + #[error("control store: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("control-store worker failed: {0}")] + Worker(#[from] tokio::task::JoinError), + #[error("control-store filesystem: {0}")] + Io(#[from] std::io::Error), + #[error("invalid control-plane data: {0}")] + Invalid(String), + #[error("control-store schema is unsupported: {0}")] + Schema(String), + #[error("snapshot generation is not newer than the current generation")] + StaleGeneration, + #[error("scheduler lease expired or was fenced by another holder")] + Fenced, +} diff --git a/crates/sqlbuild-orchestration/src/store/main/append_recall.rs b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs new file mode 100644 index 0000000000..349fe86d1a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs @@ -0,0 +1,36 @@ +//! Append Recall facts and retry-conflict evidence to the caller's control transaction. + +use chrono::SecondsFormat; +use sqlbuild_recall::records::main::conflict_row::conflict_row; +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; + +use crate::store::_helpers::recall_rows::{text, write_row}; +use crate::store::errors::ControlError; +use crate::store::models::{ControlTransaction, RecallAppendOutcome}; + +pub(crate) async fn append_recall( + transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let (mut transaction, outcome) = write_row(transaction, row).await?; + if let Some(evidence) = &outcome.conflict + && row.table != Table::Conflicts + { + let envelope = RowEnvelope { + invocation_id: text(row, "invocation_id")?.into(), + invocation_started_at: text(row, "invocation_started_at")?.into(), + producer_version: text(row, "producer_version")?.into(), + }; + let detected_at = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); + let conflict = conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + &detected_at, + ); + (transaction, _) = write_row(transaction, &conflict).await?; + } + Ok((transaction, outcome)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs new file mode 100644 index 0000000000..e97c51e077 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -0,0 +1,59 @@ +//! Serialize durable appends before allocating their storage IDs. + +use std::time::Duration; + +use chrono::Utc; +use rusqlite::OpenFlags; + +use crate::store::errors::ControlError; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{Connection, ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn begin( + store: &ControlStore, + mode: TransactionMode, +) -> Result { + let connection = match store { + ControlStore::Sqlite(path) => { + let path = path.clone(); + Connection::Sqlite( + tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch(if mode == TransactionMode::Write { + "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } else { + "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + })?; + Ok::<_, ControlError>(connection) + }) + .await??, + ) + } + ControlStore::Postgres(pool) => { + let mut transaction = pool.begin().await?; + if mode == TransactionMode::Write { + sqlx_core::query::query("SELECT pg_advisory_xact_lock(1936810572, 1)") + .execute(&mut *transaction) + .await?; + } else { + sqlx_core::query::query( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + ) + .execute(&mut *transaction) + .await?; + } + Connection::Postgres(transaction) + } + }; + let transaction = ControlTransaction { + connection: Some(connection), + now: Utc::now(), + }; + refresh_clock(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs new file mode 100644 index 0000000000..f9e3cb46e0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -0,0 +1,18 @@ +//! Commit facts and their projections together. + +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn commit(mut transaction: ControlTransaction) -> Result<(), ControlError> { + match transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))? + { + Connection::Sqlite(connection) => { + tokio::task::spawn_blocking(move || connection.execute_batch("COMMIT")).await??; + } + Connection::Postgres(transaction) => transaction.commit().await?, + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/create_location.rs b/crates/sqlbuild-orchestration/src/store/main/create_location.rs new file mode 100644 index 0000000000..075de8a140 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/create_location.rs @@ -0,0 +1,30 @@ +//! Create a server-owned location identity independent of its mutable label. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Register a new location; names are labels, never credentials or stable identifiers. +pub async fn create_location( + store: &ControlStore, + name: &str, + actor: &str, +) -> Result { + if name.trim().is_empty() || actor.trim().is_empty() { + return Err(ControlError::Invalid( + "location name and actor are required".into(), + )); + } + let mut transaction = begin(store, TransactionMode::Write).await?; + let id = Uuid::now_v7(); + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(actor.into()), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/decode_row.rs b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs new file mode 100644 index 0000000000..bc02200fb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs @@ -0,0 +1,12 @@ +//! Decode driver-neutral typed records through the store's shared row contract. + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::store::errors::ControlError; +use crate::store::types::Row; + +pub(crate) fn decode_row(row: Row) -> Result { + serde_json::from_value(Value::Object(row.into_iter().collect())) + .map_err(|error| ControlError::Invalid(error.to_string())) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs new file mode 100644 index 0000000000..b5a38030f1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -0,0 +1,70 @@ +//! Forward-only migration under the same lock used by all other control-plane writes. + +use semver::Version; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::_helpers::migration::{SCHEMA, render}; +use crate::store::_helpers::recall_schema::recall_schema; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Create the schema once, or reject drift and schemas newer than this server supports. +pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + (transaction, _) = query(transaction, &render(store, + "CREATE TABLE IF NOT EXISTS sqb_schema_migrations (version BIGINT PRIMARY KEY, \ + name TEXT NOT NULL, checksum TEXT NOT NULL, min_server_version TEXT NOT NULL, applied_at {time} NOT NULL)"), vec![]).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT version, checksum, min_server_version FROM sqb_schema_migrations ORDER BY version", + vec![], + ) + .await?; + let current = Version::parse(env!("CARGO_PKG_VERSION")) + .map_err(|error| ControlError::Schema(error.to_string()))?; + let recall_statements = recall_schema(store); + let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); + for row in &rows { + let minimum = row + .get("min_server_version") + .and_then(|value| value.as_str()) + .ok_or_else(|| { + ControlError::Schema("migration lacks a minimum server version".into()) + })?; + let minimum = + Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; + if row.get("version").and_then(|value| value.as_i64()) != Some(1) + || minimum > current + || row.get("checksum").and_then(|value| value.as_str()) != Some(digest.as_str()) + { + return Err(ControlError::Schema( + "unknown migration, checksum mismatch, or minimum server version not met".into(), + )); + } + } + if rows.is_empty() { + for statement in SCHEMA + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + for statement in recall_statements { + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ + VALUES (1, 'control_store', $1, $2, $3)", + vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs new file mode 100644 index 0000000000..ba1903c103 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -0,0 +1,12 @@ +pub(crate) mod append_recall; +pub(crate) mod begin; +pub(crate) mod commit; +pub mod create_location; +pub(crate) mod decode_row; +pub mod migrate_store; +pub mod open_store; +pub mod publish_recall; +pub(crate) mod query; +pub mod read_recall_events; +pub mod read_recall_view; +pub(crate) mod refresh_clock; diff --git a/crates/sqlbuild-orchestration/src/store/main/open_store.rs b/crates/sqlbuild-orchestration/src/store/main/open_store.rs new file mode 100644 index 0000000000..db8a450f4f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/open_store.rs @@ -0,0 +1,23 @@ +//! Open a control database; no Python or warehouse connection is involved. + +use std::time::Duration; + +use crate::store::_helpers::open_sqlite::{open_sqlite, sqlite_path}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Open PostgreSQL by URL, or SQLite by an explicit filesystem path. +pub async fn open_store(url: &str) -> Result { + if url.starts_with("postgres://") || url.starts_with("postgresql://") { + let pool = sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .acquire_timeout(Duration::from_secs(30)) + .connect(url) + .await?; + return Ok(ControlStore::Postgres(pool)); + } + let path = sqlite_path(url)?; + let configure_path = path.clone(); + tokio::task::spawn_blocking(move || open_sqlite(configure_path)).await??; + Ok(ControlStore::Sqlite(path)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs b/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs new file mode 100644 index 0000000000..bbe77db9a8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs @@ -0,0 +1,25 @@ +//! Worker publication is atomic; scheduler writes use the same append inside their tick transaction. + +use sqlbuild_recall::records::models::RecallRow; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, RecallAppendOutcome, TransactionMode}; + +/// Preserve the first copy of each key and record differing retries as Recall conflicts. +pub async fn publish_recall( + store: &ControlStore, + rows: &[RecallRow], +) -> Result, ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut outcomes = Vec::with_capacity(rows.len()); + for row in rows { + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + outcomes.push(outcome); + } + commit(transaction).await?; + Ok(outcomes) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/query.rs b/crates/sqlbuild-orchestration/src/store/main/query.rs new file mode 100644 index 0000000000..9e79cdc7a1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/query.rs @@ -0,0 +1,35 @@ +//! Execute inside an owned transaction, without blocking an async runtime on SQLite. + +use crate::store::_helpers::database::{postgres_query, sqlite_query}; +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction, Parameter}; +use crate::store::types::Row; + +pub(crate) async fn query( + mut transaction: ControlTransaction, + sql: &str, + parameters: Vec, +) -> Result<(ControlTransaction, Vec), ControlError> { + let connection = transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))?; + let rows: Vec = match connection { + Connection::Sqlite(connection) => { + let sql = sql.to_owned(); + let (connection, result) = tokio::task::spawn_blocking(move || { + let result = sqlite_query(&connection, &sql, ¶meters); + (connection, result) + }) + .await?; + transaction.connection = Some(Connection::Sqlite(connection)); + result? + } + Connection::Postgres(mut connection) => { + let result = postgres_query(&mut connection, sql, parameters).await; + transaction.connection = Some(Connection::Postgres(connection)); + result? + } + }; + Ok((transaction, rows)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs new file mode 100644 index 0000000000..ec9a4f533d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs @@ -0,0 +1,28 @@ +//! Stable cursor paging follows serialized committed appends, not producer timestamps. + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// Page events in storage order; equal producer timestamps and late arrivals remain visible. +pub async fn read_recall_events( + store: &ControlStore, + after: i64, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs new file mode 100644 index 0000000000..4c323fcd1b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs @@ -0,0 +1,46 @@ +//! Only the supported Recall views can be read; no caller-provided SQL is evaluated. + +use sqlbuild_recall::schema::constants::VIEW_NAMES; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// The API authorizes the invocation and evidence capability before entering this storage reader. +pub async fn read_recall_view( + store: &ControlStore, + (view, invocation_id): (&str, &str), + limit: u32, +) -> Result, ControlError> { + if !VIEW_NAMES.contains(&view) { + return Err(ControlError::Invalid("unknown Recall view".into())); + } + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, mut rows) = query( + transaction, + &format!("SELECT * FROM \"{view}\" WHERE invocation_id=$1 LIMIT $2"), + vec![ + Parameter::Text(invocation_id.into()), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + // These PostgreSQL view columns retain bytes: conversion to database text would reject NUL. + // Decode only the display field, leaving captured content and transcripts byte-exact. + if matches!(view, "output_tail" | "artifacts") { + for row in &mut rows { + if let Some(text) = row.get_mut("text") + && text.is_array() + { + let bytes: Vec = serde_json::from_value(text.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + *text = serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned()); + } + } + } + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs new file mode 100644 index 0000000000..0453de1ae3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs @@ -0,0 +1,26 @@ +//! Refresh the authoritative database clock after taking transaction locks. + +use chrono::DateTime; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn refresh_clock( + transaction: ControlTransaction, +) -> Result { + let sql = match transaction.connection { + Some(Connection::Postgres(_)) => "SELECT clock_timestamp() AS now", + _ => "SELECT strftime('%Y-%m-%dT%H:%M:%f000Z','now') AS now", + }; + let (mut transaction, rows) = query(transaction, sql, vec![]).await?; + let now = rows + .first() + .and_then(|row| row.get("now")) + .and_then(|value| value.as_str()) + .ok_or_else(|| ControlError::Invalid("database clock did not return a timestamp".into()))?; + transaction.now = DateTime::parse_from_rfc3339(now) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs new file mode 100644 index 0000000000..d92242c092 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; +pub(crate) mod types; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs new file mode 100644 index 0000000000..3c097f41bd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -0,0 +1,59 @@ +//! Native database handles and bound values shared by control-plane operations. + +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use sqlbuild_recall::schema::models::ColumnType; +use sqlbuild_recall::store::models::ConflictEvidence; +use uuid::Uuid; + +/// An installation's control store. Workers and locations never receive this handle. +#[derive(Clone, Debug)] +pub enum ControlStore { + Sqlite(PathBuf), + Postgres(sqlx_postgres::PgPool), +} + +/// The first copy wins; duplicate events retain their original storage cursor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecallAppendOutcome { + pub inserted: bool, + pub storage_id: Option, + pub conflict: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TransactionMode { + Read, + Write, +} + +/// An owned transaction; abandoning it rolls back, including when an async task is cancelled. +#[derive(Debug)] +pub(crate) struct ControlTransaction { + pub(crate) connection: Option, + pub(crate) now: DateTime, +} + +#[derive(Debug)] +pub(crate) enum Connection { + Sqlite(rusqlite::Connection), + Postgres(sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>), +} + +/// Parameters keep their PostgreSQL types; SQLite uses their corresponding storage classes. +#[derive(Clone, Debug)] +pub(crate) enum Parameter { + Null(ColumnType), + Text(String), + OptionalText(Option), + Bytes(Vec), + Integer(i64), + Real(f64), + Json(serde_json::Value), + Boolean(bool), + Id(Uuid), + OptionalId(Option), + Time(DateTime), + OptionalTime(Option>), +} diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql new file mode 100644 index 0000000000..c4c689ca2e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -0,0 +1,264 @@ +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id {uuid} NOT NULL, + created_at {time} NOT NULL +); +CREATE TABLE sqb_locations ( + location_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at {time} NOT NULL, + archived_at {time}, + broken_deploy_policy TEXT NOT NULL DEFAULT 'fallback' + CHECK (broken_deploy_policy IN ('fallback', 'queue')) +); +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_snapshots ( + snapshot_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + snapshot_schema_version BIGINT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at {time} NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_sources ( + snapshot_id {uuid} NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_documents ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); +CREATE TABLE sqb_current_snapshots ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL, + since {time} NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); +CREATE TABLE sqb_snapshot_changes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder {uuid} NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at {time} NOT NULL +); +CREATE TABLE sqb_jobs ( + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json {json} NOT NULL, + overlap_policy TEXT NOT NULL CHECK (overlap_policy IN ('coalesce', 'queue', 'allow')), + PRIMARY KEY (location_id, target, job_key) +); +CREATE TABLE sqb_job_nodes ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + job_key TEXT NOT NULL, + node_key TEXT NOT NULL, + PRIMARY KEY (location_id, target, job_key, node_key), + FOREIGN KEY (location_id, target, job_key) REFERENCES sqb_jobs(location_id, target, job_key) +); +CREATE TABLE sqb_triggers ( + trigger_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json {json} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id {uuid} REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); +CREATE TABLE sqb_trigger_state ( + trigger_id {uuid} PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at {time} NOT NULL, + cursor_json {json}, + last_tick_for {time}, + next_due_at {time}, + consecutive_failures BIGINT NOT NULL DEFAULT 0, + updated_at {time} NOT NULL +); +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; +CREATE TABLE sqb_trigger_state_events ( + event_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + occurred_at {time} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + request_json {json} NOT NULL +); +CREATE INDEX sqb_trigger_state_events_time ON sqb_trigger_state_events(trigger_id, occurred_at, event_id); +CREATE TABLE sqb_ticks ( + tick_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for {time} NOT NULL, + evaluated_at {time} NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'skipped', 'covered', 'failed', 'deferred')), + run_id {uuid}, + covered_by_run_id {uuid}, + reason TEXT, + details_json {json} NOT NULL, + UNIQUE (trigger_id, scheduled_for) +); +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); +CREATE TABLE sqb_runs ( + run_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id {uuid} REFERENCES sqb_runs(run_id), + root_run_id {uuid} NOT NULL, + trigger_id {uuid} REFERENCES sqb_triggers(trigger_id), + tick_id {uuid} REFERENCES sqb_ticks(tick_id), + rerun_of_run_id {uuid} REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json {json} NOT NULL, + request_json {json} NOT NULL, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled', 'presumed_failed')), + status_reason TEXT, + queued_at {time} NOT NULL, + launched_at {time}, + started_at {time}, + ended_at {time}, + cancel_requested_at {time}, + node_counts_json {json} NOT NULL, + last_event_storage_id BIGINT NOT NULL +); +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); +CREATE TABLE sqb_run_nodes ( + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + attempt BIGINT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at {time}, + ended_at {time}, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + PRIMARY KEY (run_id, node_key, attempt) +); +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (target, node_key) +); +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); +CREATE TABLE sqb_run_liveness ( + run_id {uuid} PRIMARY KEY REFERENCES sqb_runs(run_id), + worker_instance {uuid} NOT NULL, + container_id TEXT, + claimed_at {time} NOT NULL, + last_heartbeat_at {time} NOT NULL +); +CREATE TABLE sqb_principals ( + principal_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL CHECK (kind IN ('person', 'service_account')), + created_at {time} NOT NULL, + disabled_at {time} +); +CREATE TABLE sqb_access_tokens ( + token_id {uuid} PRIMARY KEY, + secret_hash TEXT NOT NULL, + principal_id {uuid} REFERENCES sqb_principals(principal_id), + location_id {uuid} REFERENCES sqb_locations(location_id), + run_id {uuid} REFERENCES sqb_runs(run_id), + claim_id {uuid}, + scope_json {json} NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time}, + CHECK ( + (principal_id IS NOT NULL AND location_id IS NULL AND run_id IS NULL AND claim_id IS NULL) + OR (principal_id IS NULL AND location_id IS NOT NULL AND run_id IS NULL AND claim_id IS NULL) + OR (principal_id IS NULL AND location_id IS NULL AND run_id IS NOT NULL AND claim_id IS NOT NULL) + ) +); +CREATE INDEX sqb_access_tokens_principal ON sqb_access_tokens(principal_id); +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); +CREATE TABLE sqb_security_audit ( + event_id {uuid} PRIMARY KEY, + occurred_at {time} NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id {uuid} NOT NULL, + ip_address TEXT, + scope_json {json} NOT NULL, + before_json {json} NOT NULL, + after_json {json} NOT NULL +); +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/store/types.rs b/crates/sqlbuild-orchestration/src/store/types.rs new file mode 100644 index 0000000000..c1df9d2d41 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/types.rs @@ -0,0 +1,3 @@ +//! Internal result rows use native JSON scalars, with UUIDs and timestamps as strings. + +pub(crate) type Row = std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs new file mode 100644 index 0000000000..9ee75a0eb6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -0,0 +1,12 @@ +#[path = "api/helpers.rs"] +mod helpers; +#[path = "runs/helpers.rs"] +mod run_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "api/test_contract.rs"] +mod test_contract; +#[path = "api/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs new file mode 100644 index 0000000000..395e1535f5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::test_types::TestApi; + +pub(crate) async fn start_api(store: &ControlStore) -> TestApi { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated API socket"); + let url = format!("http://{}", listener.local_addr().expect("API address")); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let store = store.clone(); + let task = tokio::spawn(serve_api(store, listener, async { + let _ = signal.await; + })); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + break; + } + assert!(!task.is_finished(), "API failed during startup"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("API becomes ready"); + TestApi { + url, + shutdown, + task, + } +} + +pub(crate) async fn stop_api(api: TestApi) { + api.shutdown.send(()).expect("request API shutdown"); + tokio::time::timeout(Duration::from_secs(2), api.task) + .await + .expect("prompt API shutdown") + .expect("join API server") + .expect("API server stopped cleanly"); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs new file mode 100644 index 0000000000..ef7d0d0902 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -0,0 +1,331 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunStatus}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::run_helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; + +#[test] +fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("another location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let expiry = Utc::now() + chrono::Duration::hours(1); + let location_token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location credential"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let path = format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + api.url + ); + let response = client + .post(&path) + .header("Content-Type", "application/json") + .body("invalid-json") + .send() + .await + .expect("unauthenticated request"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "authentication precedes JSON parsing" + ); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI response") + .json() + .await + .expect("OpenAPI JSON"); + assert_eq!( + openapi["components"]["securitySchemes"]["bearer"]["scheme"], + "bearer" + ); + assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); + assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); + let input = snapshot(location_id); + let response = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&input) + .send() + .await + .expect("publish through API"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let published: SnapshotRecord = response.json().await.expect("snapshot response"); + let response = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&input) + .send() + .await + .expect("stale publish"); + assert_eq!(response.status(), StatusCode::CONFLICT); + let response = client + .post(format!( + "{}/api/locations/{other}/targets/development/snapshots", + api.url + )) + .bearer_auth(&location_token.bearer) + .json(&snapshot(other)) + .send() + .await + .expect("cross-location publish"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let current: serde_json::Value = client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("current snapshot") + .json() + .await + .expect("snapshot JSON"); + assert_eq!( + current["record"]["snapshot_id"], + published.snapshot_id.to_string() + ); + assert_eq!(current["nodes"]["orders"]["compiled_sql"], "select 1 as id"); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let run_id = claim.request.run_id; + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker credential"); + let events_path = format!("{}/api/runs/{run_id}/events", api.url); + let heartbeat_path = format!("{}/api/runs/{run_id}/heartbeat", api.url); + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&location_token.bearer) + .json(&started) + .send() + .await + .expect("location cannot act as worker") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .get(&path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker cannot read location snapshot") + .status(), + StatusCode::FORBIDDEN + ); + for applied in [true, false] { + let response = client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&started) + .send() + .await + .expect("report started"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("report response"), + json!({"applied": applied}) + ); + } + let wrong_claim = RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&wrong_claim) + .send() + .await + .expect("wrong claim") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{}/api/runs/{}/heartbeat", api.url, Uuid::now_v7())) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("other run") + .status(), + StatusCode::FORBIDDEN + ); + let cancellation = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-admin".into(), + }, + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&cancellation) + .send() + .await + .expect("worker cannot impersonate admin") + .status(), + StatusCode::BAD_REQUEST + ); + let response = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker heartbeat"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("heartbeat response"), + json!({"cancel_requested": false}) + ); + change_run(store, RunAuthority::Operator("test-admin"), &cancellation) + .await + .expect("operator cancellation"); + let response: serde_json::Value = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("cancel heartbeat") + .json() + .await + .expect("cancel response"); + assert_eq!(response, json!({"cancel_requested": true})); + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: sqlbuild_orchestration::runs::models::RunOutcome::Cancelled, + reason: "worker acknowledged cancellation".into(), + exit_code: Some(143), + }, + ..started + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&terminal) + .send() + .await + .expect("worker terminates") + .status(), + StatusCode::OK + ); + assert_eq!( + client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("run projection")[0] + .status, + RunStatus::Cancelled + ); + revoke_token(store, &audit, location_token.identity.token_id) + .await + .expect("revoke location token"); + assert_eq!( + client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("revocation effective") + .status(), + StatusCode::UNAUTHORIZED + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs new file mode 100644 index 0000000000..d62cdc25ce --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -0,0 +1,7 @@ +use sqlbuild_orchestration::store::errors::ControlError; + +pub(crate) struct TestApi { + pub(crate) url: String, + pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, + pub(crate) task: tokio::task::JoinHandle>, +} diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs new file mode 100644 index 0000000000..a4b17699db --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials.rs @@ -0,0 +1,12 @@ +#[path = "ticks/fault_helpers.rs"] +mod fault_helpers; +#[path = "runs/helpers.rs"] +mod run_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "credentials/test_contract.rs"] +mod test_contract; +#[path = "ticks/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs new file mode 100644 index 0000000000..6768c12897 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -0,0 +1,241 @@ +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use sqlbuild_orchestration::credentials::main::authenticate_token::authenticate_token; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::run_helpers::{finish, queue, request, snapshot}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +#[test] +fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("loopback")), + }; + let fault = TickFaultTestCase { + description: "security audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!( + create_principal( + store, + &context, + ("orders-ci", PrincipalKind::ServiceAccount) + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + let principal_id = create_principal( + store, + &context, + ("orders-ci", PrincipalKind::ServiceAccount), + ) + .await + .expect("principal creation and audit rolled back together"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let scopes = [ + CredentialScope::Principal { principal_id }, + CredentialScope::Location { location_id }, + ]; + let expiry = Utc::now() + Duration::hours(1); + for scope in scopes { + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("issue scoped token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("replica verifies"), + Some(issued.identity.clone()) + ); + assert!( + !format!("{issued:?}").contains(&issued.bearer), + "debug output redacts secrets" + ); + let replacement = if issued.bearer.ends_with('0') { + "1" + } else { + "0" + }; + let tampered = + format!("{}{replacement}", &issued.bearer[..issued.bearer.len() - 1]); + for invalid in [ + &tampered, + "", + "not-a-token", + &format!("{}.{}", Uuid::now_v7(), "0".repeat(64)), + ] { + assert!( + authenticate_token(&replica, invalid) + .await + .expect("invalid credential") + .is_none() + ); + } + install_fault(store, &fault).await; + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("failed revocation rolled back") + .is_some() + ); + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .expect("revoke") + ); + assert!( + !revoke_token(&replica, &context, issued.identity.token_id) + .await + .expect("retry revoke") + ); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("revoked token") + .is_none() + ); + assert!( + issue_token(store, &context, scope, Utc::now() - Duration::seconds(1)) + .await + .is_err() + ); + let stored: Vec = match store { + ControlStore::Sqlite(path) => { + let connection = + rusqlite::Connection::open(path).expect("inspect durable tokens"); + connection + .prepare("SELECT secret_hash FROM sqb_access_tokens") + .expect("token hashes") + .query_map([], |row| row.get(0)) + .expect("read token hashes") + .collect::>() + .expect("hashes") + } + ControlStore::Postgres(pool) => sqlx_core::query_scalar::query_scalar( + "SELECT secret_hash FROM sqb_access_tokens", + ) + .fetch_all(pool) + .await + .expect("read durable token hashes"), + }; + assert!( + stored + .iter() + .all(|hash| hash.len() == 64 && hash != &issued.bearer) + ); + } + for scope in [ + CredentialScope::Principal { + principal_id: Uuid::now_v7(), + }, + CredentialScope::Location { + location_id: Uuid::now_v7(), + }, + CredentialScope::Run { + run_id: Uuid::now_v7(), + claim_id: Uuid::now_v7(), + }, + ] { + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "inactive scopes cannot acquire credentials" + ); + } + let published = publish_snapshot(store, &snapshot(location_id)) + .await + .expect("snapshot"); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let scope = CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("mint claimed-run token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("worker token"), + Some(issued.identity.clone()) + ); + assert!( + issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: Uuid::now_v7() + }, + expiry + ) + .await + .is_err() + ); + finish(store, &lease, &claim).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("terminal token expires immediately") + .is_none() + ); + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "a terminal claim cannot mint a new token" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/recall.rs b/crates/sqlbuild-orchestration/tests/recall.rs new file mode 100644 index 0000000000..274b96b9ed --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall.rs @@ -0,0 +1,12 @@ +#[path = "recall/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "recall/test_contract.rs"] +mod test_contract; +#[path = "recall/test_types.rs"] +mod test_types; +#[path = "recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/recall/helpers.rs new file mode 100644 index 0000000000..72e824e82a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/helpers.rs @@ -0,0 +1,137 @@ +use serde_json::json; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::publish_recall::publish_recall; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; + +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; + +pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { + event_object_row(&RowEnvelope { invocation_id: "orders-build".into(), + invocation_started_at: "2026-03-01T00:30:00+01:00".into(), producer_version: "test".into() }, + json!({ "event_id": id, "event_type": "invocation_completed", "schema_version": 1, + "occurred_at": occurred_at, "producer": "worker", "invocation_sequence": 1, + "payload": { "duration_ms": duration_ms, "exit_code": 0, "nested": [null, true, "orders"] } }) + .as_object().expect("event object")).expect("valid lifecycle event") +} + +pub(crate) async fn recall_contract(backend: Backend) -> Option> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store) + .await + .expect("migrate control and Recall schema"); + let second = second_store(store).await; + let first = event("orders-first", "2026-03-01T00:00:00Z", 3.25); + let first_batch = [first.clone()]; + let (left, right) = tokio::join!( + publish_recall(store, &first_batch), + publish_recall(&second, &first_batch) + ); + let left = left.expect("first writer").remove(0); + let right = right.expect("second writer").remove(0); + assert_ne!(left.inserted, right.inserted, "exactly one writer inserts"); + assert_eq!( + left.storage_id, right.storage_id, + "duplicates preserve cursor" + ); + let cursor = left.storage_id.expect("event storage cursor"); + let conflict = publish_recall(store, &[event("orders-first", "2026-03-01T00:00:00Z", 7.5)]) + .await + .expect("record conflicting retry") + .remove(0); + assert!( + conflict.conflict.is_some(), + "different content records conflict" + ); + let later = event("orders-later-arrival", "2026-02-01T00:00:00Z", 2.5); + let malformed = event("orders-malformed", "2026-04-01T00:00:00Z", 1.0); + let mut malformed = malformed; + malformed.values.retain(|(name, _)| *name != "event_id"); + assert!( + publish_recall(store, &[later.clone(), malformed]) + .await + .is_err(), + "whole malformed batch rolls back" + ); + assert!( + read_recall_events(store, cursor, 100) + .await + .expect("rolled-back page") + .is_empty() + ); + publish_recall(store, &[later]) + .await + .expect("retry rolled-back event"); + let page = read_recall_events(store, cursor, 1) + .await + .expect("late arrival page"); + assert_eq!(page.len(), 1); + assert_eq!( + page[0]["event_id"], "orders-later-arrival", + "page follows commit order, not producer time" + ); + let rows = read_recall_events(store, 0, 100).await.expect("all events"); + assert_eq!( + rows[0]["duration_ms"], + json!(3.25), + "first content wins with fractional precision" + ); + let payload = rows[0]["payload"].as_str().map_or_else( + || rows[0]["payload"].clone(), + |text| serde_json::from_str(text).expect("JSON payload"), + ); + assert_eq!(payload["nested"], json!([null, true, "orders"])); + let artifact = ArtifactInput { + digest: "sha256:orders".into(), + kind: "sql".into(), + media_type: "application/octet-stream".into(), + content: vec![0, 255, 1, 0], + seen_at: "2026-01-01T00:00:00Z".into(), + ref_id: "orders".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }; + let first_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "first-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + publish_recall(store, &first_artifact) + .await + .expect("first binary artifact"); + let artifact = ArtifactInput { + seen_at: "2026-04-01T00:00:00Z".into(), + ..artifact + }; + let second_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "second-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + let outcomes = publish_recall(store, &second_artifact) + .await + .expect("same artifact across months"); + assert!( + !outcomes[0].inserted && outcomes[0].conflict.is_none(), + "artifact identity is global, not partition-local" + ); + assert!(outcomes[1].inserted, "each invocation keeps its reference"); + let ids = rows + .iter() + .map(|row| row["event_id"].as_str().expect("event ID").to_owned()) + .collect(); + close_store(second).await; + close_test_store(fixture).await; + Some(ids) +} diff --git a/crates/sqlbuild-orchestration/tests/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/recall/test_contract.rs new file mode 100644 index 0000000000..d8d8bf5629 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/test_contract.rs @@ -0,0 +1,28 @@ +use crate::helpers::recall_contract; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::RecallTestCase; + +#[test] +fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { + let test_cases = [ + RecallTestCase { + description: "SQLite Recall transactions", + backend: Backend::Sqlite, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + RecallTestCase { + description: "PostgreSQL Recall transactions", + backend: Backend::Postgres, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + ]; + for test_case in test_cases { + let observed = run(recall_contract(test_case.backend)); + assert!( + observed.is_none_or(|ids| ids == test_case.expected_event_ids), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/recall/test_types.rs new file mode 100644 index 0000000000..8ecb5f4f63 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/test_types.rs @@ -0,0 +1,13 @@ +use crate::store_test_types::Backend; + +pub(crate) struct RecallTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_event_ids: Vec<&'static str>, +} + +pub(crate) struct RecallViewTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_complete: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/recall/test_views.rs new file mode 100644 index 0000000000..0dd0f229a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/recall/test_views.rs @@ -0,0 +1,192 @@ +use serde_json::json; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::publish_recall::publish_recall; +use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::delivery_row::delivery_row; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::heartbeat_row::heartbeat_row; +use sqlbuild_recall::records::main::invocation_row::invocation_row; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{ + ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, +}; + +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::RecallViewTestCase; + +#[test] +fn given_native_context_output_and_audits_when_reading_views_then_both_databases_preserve_their_meaning() + { + let test_cases = [ + RecallViewTestCase { + description: "SQLite supported views", + backend: Backend::Sqlite, + expected_complete: 1, + }, + RecallViewTestCase { + description: "PostgreSQL supported views", + backend: Backend::Postgres, + expected_complete: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate view store"); + let envelope = RowEnvelope { + invocation_id: "orders-output".into(), + invocation_started_at: "2026-01-02T00:00:00Z".into(), + producer_version: "test".into(), + }; + let context = InvocationContext { + command: Some("build".into()), + selection: vec!["orders".into()], + external_context: Some( + json!({"integration": {"name": "orders", "retry_number": 2}}).to_string(), + ), + ..InvocationContext::default() + }; + let mut rows = vec![ + invocation_row(&envelope, &context), + heartbeat_row( + &envelope, + ("2026-01-02T00:00:01Z", "2026-01-02T00:00:31Z"), + (None, None), + (Some("build"), None), + ), + ]; + for (sequence, (stream, bytes)) in [ + (OutputStream::Stdout, b"orders\x00\xff".as_slice()), + (OutputStream::Stderr, b"warning\n".as_slice()), + (OutputStream::Stdout, b"\xc3\x28 tail".as_slice()), + ] + .into_iter() + .enumerate() + { + rows.push(output_row( + &envelope, + &OutputChunk { + sequence: sequence as u64, + stream, + occurred_at: "2026-01-02T00:00:02Z".into(), + data: bytes.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + )); + } + rows.push(delivery_row( + &envelope, + "output", + ( + &DeliveryCounts { + accepted: 3, + delivered: 3, + ..DeliveryCounts::default() + }, + true, + ), + "2026-01-02T00:00:03Z", + )); + for (event_id, event_type, payload) in [ + ( + "orders-audit", + "audit_completed", + json!({"audit_name":"unique_orders", "violation_count": 7, "measured_value": 3.5, "target_name":"orders", "evidence_truncated": true}), + ), + ( + "orders-complete", + "invocation_completed", + json!({"duration_ms": 12.75, "exit_code": 0}), + ), + ] { + rows.push(event_object_row(&envelope, json!({"event_id": event_id, "event_type": event_type, "schema_version": 1, "occurred_at": "2026-01-02T00:00:03Z", "producer": "worker", "invocation_sequence": 1, "payload": payload}).as_object().expect("event object")).expect("lifecycle fact")); + } + rows.extend(artifact_rows( + &envelope, + &ArtifactInput { + digest: "sha256:orders-display".into(), + kind: "sql".into(), + media_type: "text/plain".into(), + content: b"orders\x00\xff".to_vec(), + seen_at: envelope.invocation_started_at.clone(), + ref_id: "orders-statement".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }, + )); + publish_recall(store, &rows) + .await + .expect("write view inputs"); + let output = + read_recall_view(store, ("output_invocations", &envelope.invocation_id), 100) + .await + .expect("output accounting"); + assert_eq!(output[0]["complete"], json!(test_case.expected_complete)); + assert_eq!(output[0]["chunks"], json!(3)); + assert_eq!(output[0]["missing_sequences"], json!(0)); + let transcript = + read_recall_view(store, ("output_transcripts", &envelope.invocation_id), 100) + .await + .expect("binary transcript"); + assert_eq!( + transcript[0]["stdout"], + json!([b"orders\x00\xff".as_slice(), b"\xc3\x28 tail".as_slice()].concat()) + ); + assert_eq!( + transcript[0]["combined"], + json!( + [ + b"orders\x00\xff".as_slice(), + b"warning\n".as_slice(), + b"\xc3\x28 tail".as_slice() + ] + .concat() + ) + ); + let invocation = read_recall_view(store, ("invocations", &envelope.invocation_id), 100) + .await + .expect("invocation projection"); + assert_eq!(invocation[0]["status"], "completed"); + assert_eq!(invocation[0]["duration_ms"], json!(12.75)); + assert_eq!(invocation[0]["integration_retry"], json!(2)); + let audit = read_recall_view(store, ("audit_results", &envelope.invocation_id), 100) + .await + .expect("audit projection"); + assert_eq!(audit[0]["violation_count"], json!(7)); + assert_eq!(audit[0]["measured_value"], json!(3.5)); + assert_eq!(audit[0]["evidence_truncated"], json!(1)); + let tail = read_recall_view(store, ("output_tail", &envelope.invocation_id), 100) + .await + .expect("lossy output display"); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 0) + .expect("first chunk")["text"], + json!("orders\u{0}\u{fffd}"), + "{}", + test_case.description + ); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 2) + .expect("last chunk")["text"], + json!("\u{fffd}( tail") + ); + let artifacts = read_recall_view(store, ("artifacts", &envelope.invocation_id), 100) + .await + .expect("lossy artifact display"); + assert_eq!(artifacts[0]["text"], json!("orders\u{0}\u{fffd}")); + assert_eq!(artifacts[0]["content"], json!(b"orders\x00\xff".to_vec())); + assert_eq!(artifacts[0]["available"], json!(1)); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs new file mode 100644 index 0000000000..f2b42df0c2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -0,0 +1,18 @@ +#[path = "ticks/fault_helpers.rs"] +mod fault_helpers; +#[path = "runs/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "runs/test_claims.rs"] +mod test_claims; +#[path = "runs/test_coalesce.rs"] +mod test_coalesce; +#[path = "runs/test_faults.rs"] +mod test_faults; +#[path = "runs/test_lifecycle.rs"] +mod test_lifecycle; +#[path = "ticks/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs new file mode 100644 index 0000000000..e98f62b1df --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -0,0 +1,118 @@ +//! Run fixtures exercise native queues and Recall against isolated real databases. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::json; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunKind, RunOrigin, RunOutcome, RunRequest, +}; +use sqlbuild_orchestration::scheduler::models::{Overlap, SchedulerLease}; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: ["customers", "inventory", "orders", "products"] + .into_iter() + .map(|name| { + ( + name.into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + ) + }) + .collect(), + documents: BTreeMap::new(), + } +} + +pub(crate) fn request( + (location_id, snapshot_id): (Uuid, Uuid), + id: u128, + priority: i64, + nodes: &[&str], +) -> RunRequest { + let selection: BTreeSet = nodes.iter().map(|name| (*name).into()).collect(); + let run_id = Uuid::from_u128(id); + RunRequest { + run_id, + location_id, + target: "development".into(), + snapshot_id, + image_id: Some("sha256:synthetic-orders-image".into()), + kind: RunKind::Adhoc, + origin: RunOrigin::Manual, + job_key: None, + parent_run_id: None, + root_run_id: run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: "test-operator".into(), + priority, + write_set: selection.clone(), + selection, + settings: BTreeMap::new(), + concurrency_groups: BTreeSet::new(), + overlap: Overlap::Allow, + } +} + +pub(crate) async fn queue(store: &ControlStore, request: &RunRequest) { + assert!( + request_run(store, request) + .await + .expect("queue pinned request") + ); + assert!( + !request_run(store, request) + .await + .expect("retry pinned request") + ); + let mut changed = request.clone(); + changed.priority += 1; + assert!( + request_run(store, &changed).await.is_err(), + "a reused run ID cannot change its request" + ); +} + +pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "synthetic worker exit".into(), + exit_code: Some(1), + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("terminate claimed work") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("retry terminal fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs new file mode 100644 index 0000000000..4489f0322d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -0,0 +1,252 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::RunStatus; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::helpers::{finish, queue, request, snapshot}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::store_test_types::Backend; + +#[test] +fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + for input in [ + request(scope, 1, 1, &["orders"]), + request(scope, 2, 3, &["orders"]), + request(scope, 3, 2, &["products"]), + ] { + queue(store, &input).await; + } + let second = second_store(store).await; + let limits = BTreeMap::new(); + let (first, other) = tokio::join!( + claim_next_run(store, &lease, &limits), + claim_next_run(&second, &lease, &limits) + ); + let first = first.expect("first claimant").expect("claim available"); + let other = other + .expect("second claimant") + .expect("independent claim available"); + let claimed: std::collections::BTreeSet<_> = + [first.request.run_id, other.request.run_id] + .into_iter() + .collect(); + assert_eq!( + claimed, + [Uuid::from_u128(2), Uuid::from_u128(3)] + .into_iter() + .collect(), + "higher priority wins the shared write set" + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("blocked run stays queued") + .is_none() + ); + queue(store, &request(scope, 4, 9, &["customers", "orders"])).await; + queue(store, &request(scope, 5, 0, &["customers"])).await; + let independent = claim_next_run(store, &lease, &limits) + .await + .expect("try partially conflicting candidate") + .expect("independent work proceeds"); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(5), + "customers was not leaked by the failed customers/orders claim" + ); + let orders = [&first, &other] + .into_iter() + .find(|claim| claim.request.run_id == Uuid::from_u128(2)) + .expect("orders owner"); + finish(store, &lease, orders).await; + finish(store, &lease, &independent).await; + let next = claim_next_run(store, &lease, &limits) + .await + .expect("terminal cleanup committed") + .expect("next priority run claims both nodes"); + assert_eq!(next.request.run_id, Uuid::from_u128(4)); + let events = read_recall_events(store, 0, 100) + .await + .expect("read immutable run stream"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_claimed") + .count(), + 4 + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated") + .count(), + 2 + ); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("read atomic projections"); + assert_eq!( + runs.iter() + .filter(|run| run.status == RunStatus::Failed) + .count(), + 2 + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_still_count_and_terminal_cleanup_unblocks_work() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(2).expect("nonzero limit"), + )]); + for id in 1..=3 { + let mut input = request((location, published.snapshot_id), id, 0, &["orders"]); + input.write_set.clear(); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + } + assert!( + claim_next_run(store, &lease, &BTreeMap::new()) + .await + .is_err(), + "unknown groups cannot silently become unlimited" + ); + let first = claim_next_run(store, &lease, &limits) + .await + .expect("first slot") + .expect("available"); + let second = claim_next_run(store, &lease, &limits) + .await + .expect("second slot") + .expect("available"); + assert_eq!(first.request.run_id, Uuid::from_u128(1)); + assert_eq!(second.request.run_id, Uuid::from_u128(2)); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("group full") + .is_none() + ); + finish(store, &lease, &first).await; + limits.insert( + "warehouse".into(), + NonZeroU32::new(1).expect("nonzero lowered limit"), + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("high slot counts after limit reduction") + .is_none() + ); + finish(store, &lease, &second).await; + let third = claim_next_run(store, &lease, &limits) + .await + .expect("released group") + .expect("third queued run"); + assert_eq!(third.request.run_id, Uuid::from_u128(3)); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_independent_work_is_not_starved() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let mut lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + queue(store, &request(scope, 1, 0, &["orders"])).await; + claim_next_run(store, &lease, &limits) + .await + .expect("claim blocker") + .expect("available"); + for id in 2..=104 { + queue(store, &request(scope, id, 0, &["orders"])).await; + if id % 20 == 0 { + lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( + store, &lease, + ) + .await + .expect("keep leader live while preparing backlog"); + } + } + queue(store, &request(scope, 105, -1, &["customers"])).await; + let started = std::time::Instant::now(); + let independent = claim_next_run(store, &lease, &limits) + .await + .expect("scan past a full blocked page") + .expect("independent work"); + assert_eq!(independent.request.run_id, Uuid::from_u128(105)); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "claim scan exceeded five seconds on {backend:?}" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs new file mode 100644 index 0000000000..c05d357583 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -0,0 +1,103 @@ +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::scheduler::models::TickDecision; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; + +#[test] +fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2030-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("first calendar publication"); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger) + .await + .expect("first tick"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim first run") + .expect("scheduled run"); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + input.generation = 2; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh calendar"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("second tick") + .expect("active trigger"); + assert_eq!(plan.actions.len(), 1); + let TickDecision::Requested { run_id: waiting } = plan.actions[0].decision else { + panic!("future work must not be covered by an already claimed run"); + }; + assert_ne!(waiting, claim.request.run_id); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!([ + "2020-01-01T00:00:00Z", + "2020-01-02T00:00:00Z", + "2020-01-03T00:00:00Z" + ]); + publish_snapshot(store, &input) + .await + .expect("next calendar refresh"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("third tick") + .expect("active trigger"); + assert_eq!( + plan.actions[0].decision, + TickDecision::Covered { + covered_by_run_id: waiting + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("one executing and one waiting") + .len(), + 2 + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs new file mode 100644 index 0000000000..7ab8721043 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -0,0 +1,162 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; +use crate::helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +#[test] +fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let mut input = request( + (location, published.snapshot_id), + 1, + 1, + &["customers", "orders"], + ); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(1).expect("positive limit"), + )]); + let facts = read_recall_events(store, 0, 100) + .await + .expect("requested and queued facts"); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("queued projection"); + for table in [ + "sqb_write_claims", + "sqb_concurrency_slots", + "sqb_run_liveness", + "recall_events", + "sqb_runs", + ] { + let point = TickFaultTestCase { + description: "claim publication failure", + table, + operation: if table == "sqb_runs" { + "UPDATE" + } else { + "INSERT" + }, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected claim failure')", + postgres_body: "RAISE EXCEPTION 'injected claim failure'", + }; + install_fault(store, &point).await; + assert!( + claim_next_run(store, &lease, &limits).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back claim fact"), + facts + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("rolled back claim projection"), + projected + ); + } + let expiry = TickFaultTestCase { + description: "claim fence expires before commit", + table: "sqb_run_liveness", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }; + install_fault(store, &expiry).await; + assert!( + matches!( + claim_next_run(store, &lease, &limits).await, + Err(ControlError::Fenced) + ), + "{}", + expiry.description + ); + remove_fault(store, &expiry).await; + let claim = claim_next_run(store, &lease, &limits) + .await + .expect("retry complete claim") + .expect("no leaked claims or slots"); + assert_eq!(claim.request, input); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("already claimed run is not relaunched") + .is_none() + ); + expire_lease(store).await; + let next = acquire_lease(store, Uuid::now_v7()) + .await + .expect("take over") + .expect("new leader"); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + }; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &launched).await, + Err(ControlError::Fenced) + ), + "a former leader cannot acknowledge a launch after takeover" + ); + assert!( + change_run(store, RunAuthority::Scheduler(&next), &launched) + .await + .expect("successor reconciles the existing claim") + ); + assert!( + !heartbeat_run(store, (input.run_id, claim.claim_id)) + .await + .expect("workers remain alive through scheduler failover") + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs new file mode 100644 index 0000000000..7ed2c6874a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -0,0 +1,427 @@ +use std::collections::BTreeMap; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; +use sqlbuild_orchestration::runs::main::list_run_nodes::list_run_nodes; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +#[test] +fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue(store, &request(scope, 1, 2, &["customers", "orders"])).await; + queue(store, &request(scope, 2, 1, &["customers", "orders"])).await; + let limits = BTreeMap::new(); + let claim = claim_next_run(store, &lease, &limits) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + let other_worker = RunAuthority::Worker { + run_id: Uuid::from_u128(2), + claim_id: Uuid::now_v7(), + }; + assert!(change_run(store, other_worker, &started).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator("test-operator"), &started) + .await + .is_err() + ); + assert!( + change_run(store, worker, &started) + .await + .expect("worker can start before launcher acknowledgement") + ); + assert!( + !change_run(store, worker, &started) + .await + .expect("identical retry") + ); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &launched) + .await + .expect("late launcher acknowledgement"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .iter() + .find(|run| run.run_id == claim.request.run_id) + .expect("started run") + .status, + RunStatus::Running + ); + let success = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Success, + reason: "worker complete".into(), + exit_code: Some(0), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &success).await.is_err(), + "unfinished nodes cannot be marked successful" + ); + for (node, outcome) in [ + ("customers", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ] { + let node_start = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeStarted { + node_key: node.into(), + }, + ..started.clone() + }; + change_run(store, worker, &node_start) + .await + .expect("node started"); + let completed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeFinished { + node_key: node.into(), + outcome, + rows_affected: Some(17), + error_code: (outcome == NodeOutcome::Failed).then(|| "E100".into()), + skipped_because_of: None, + }, + ..started.clone() + }; + if node == "customers" { + let before_nodes = list_run_nodes(store, started.run_id) + .await + .expect("node before fault"); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("events before node fault"); + for table in ["sqb_run_nodes", "sqb_runs"] { + let point = TickFaultTestCase { + description: "node projection and counter failure", + table, + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &completed).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_run_nodes(store, started.run_id) + .await + .expect("node rollback"), + before_nodes + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("node fact rollback"), + before_events + ); + } + } + assert!( + change_run(store, worker, &completed) + .await + .expect("node completed") + ); + assert!( + !change_run(store, worker, &completed) + .await + .expect("node report retry") + ); + } + assert!( + change_run(store, worker, &success).await.is_err(), + "mixed node outcomes are partial, not success" + ); + let partial = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Partial, + reason: "independent customer work completed".into(), + exit_code: Some(1), + }, + ..started.clone() + }; + let before = read_recall_events(store, 0, 100) + .await + .expect("before terminal"); + for table in [ + "recall_events", + "sqb_runs", + "sqb_write_claims", + "sqb_run_liveness", + ] { + let point = TickFaultTestCase { + description: "terminal transaction failure", + table, + operation: if table == "recall_events" { + "INSERT" + } else if table.starts_with("sqb_run") && table != "sqb_run_liveness" { + "UPDATE" + } else { + "DELETE" + }, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected terminal failure')", + postgres_body: "RAISE EXCEPTION 'injected terminal failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &partial).await.is_err(), + "{} at {table} must roll back the complete transition", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back event"), + before + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("claims retained") + .is_none() + ); + } + let unresponsive = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Unresponsive, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &unresponsive) + .await + .expect("monitor records loss of heartbeat"); + assert!( + !heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat recovers the worker") + ); + assert!( + heartbeat_run(store, (started.run_id, Uuid::now_v7())) + .await + .is_err(), + "another claim cannot report this run alive" + ); + let cancel = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &cancel).await.is_err(), + "workers cannot impersonate operators" + ); + change_run(store, RunAuthority::Operator("test-operator"), &cancel) + .await + .expect("record cancellation"); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat reports pending cancellation") + ); + assert!( + claim_next_run(store, &lease, &limits) + .await + .expect("cancel request does not release an executing run's claims") + .is_none() + ); + assert!( + change_run(store, worker, &partial) + .await + .expect("terminal report atomically releases claims") + ); + assert!( + !change_run(store, worker, &partial) + .await + .expect("terminal retry after liveness cleanup") + ); + let mut changed = partial.clone(); + changed.change = RunChange::Finished { + outcome: RunOutcome::Success, + reason: "changed retry".into(), + exit_code: Some(0), + }; + assert!( + change_run(store, worker, &changed).await.is_err(), + "same event ID cannot change its content" + ); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .is_err(), + "terminal runs cannot be revived by a late heartbeat" + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"); + assert_eq!(projected.status, RunStatus::Partial); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("success".into(), 1), ("failed".into(), 1)]) + ); + let nodes = list_run_nodes(store, started.run_id) + .await + .expect("node evidence"); + assert_eq!(nodes[0].node_key, "customers"); + assert_eq!(nodes[0].status, "success"); + assert_eq!(nodes[0].rows_affected, Some(17)); + assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let following = claim_next_run(store, &lease, &limits) + .await + .expect("claim following run") + .expect("released capacity"); + assert_eq!(following.request.run_id, Uuid::from_u128(2)); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_launch_and_preserves_the_actor() + { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + queue( + store, + &request((location, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), + claim_id: None, + change: RunChange::CancelRequested { + actor: "test-operator".into(), + }, + }; + assert!( + change_run(store, RunAuthority::Operator("another-operator"), &request) + .await + .is_err() + ); + assert!( + change_run(store, RunAuthority::Operator("test-operator"), &request) + .await + .expect("cancel before launch") + ); + assert!( + !change_run(store, RunAuthority::Operator("test-operator"), &request) + .await + .expect("retry cancellation") + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + assert!( + claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("cancelled work cannot launch") + .is_none() + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("cancelled projection") + .remove(0); + assert_eq!(projected.status, RunStatus::Cancelled); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("cancelled".into(), 1)]) + ); + let events = read_recall_events(store, 0, 100) + .await + .expect("immutable cancellation facts"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("type")) + .collect::>(), + [ + "run_requested", + "run_queued", + "run_cancel_requested", + "run_terminated" + ] + ); + assert_eq!( + events[2]["payload"]["change"]["change"]["actor"], + "test-operator" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs new file mode 100644 index 0000000000..159c963ed1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -0,0 +1,10 @@ +#[path = "scheduler/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "scheduler/test_contract.rs"] +mod test_contract; +#[path = "scheduler/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs new file mode 100644 index 0000000000..f1711e4bfc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -0,0 +1,91 @@ +//! Contention and expiry exercise public APIs against the real databases, not fake clocks. + +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; + +async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + panic!("expected SQLite fixture"); + }; + rusqlite::Connection::open(path) + .expect("open fault injection connection") + .execute( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + [], + ) + .expect("expire SQLite scheduler lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); +} + +pub(crate) async fn leadership_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let store = &test_store.store; + migrate_store(store) + .await + .expect("migrate leadership store"); + let second = second_store(store).await; + let (first, other) = tokio::join!( + acquire_lease(store, Uuid::now_v7()), + acquire_lease(&second, Uuid::now_v7()) + ); + let first = first.expect("first contender completes"); + let other = other.expect("second contender completes"); + assert_ne!( + first.is_some(), + other.is_some(), + "exactly one concurrent contender is elected" + ); + let lease = first.or(other).expect("one elected holder"); + let renewed = renew_lease(&second, &lease) + .await + .expect("live epoch renews"); + assert_eq!(renewed.epoch, lease.epoch); + assert!(renewed.expires_at >= lease.expires_at); + assert!( + acquire_lease(store, lease.holder) + .await + .expect("duplicate acquisition completes") + .is_none() + ); + let mut forged = lease.clone(); + forged.holder = Uuid::now_v7(); + assert!(matches!( + renew_lease(store, &forged).await, + Err(ControlError::Fenced) + )); + expire_lease(store).await; + assert!( + matches!(renew_lease(store, &lease).await, Err(ControlError::Fenced)), + "even an uncontested expired holder must reacquire a new epoch" + ); + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("acquire expired lease") + .expect("successor elected"); + assert!(matches!( + renew_lease(store, &lease).await, + Err(ControlError::Fenced) + )); + renew_lease(store, &successor) + .await + .expect("successor owns the current fence"); + let epoch = successor.epoch; + close_store(second).await; + close_test_store(test_store).await; + Some(epoch) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs new file mode 100644 index 0000000000..084f5fbb10 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs @@ -0,0 +1,28 @@ +use crate::helpers::leadership_contract; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::LeadershipTestCase; + +#[test] +fn given_two_schedulers_when_leadership_expires_then_only_the_new_epoch_can_renew() { + let test_cases = [ + LeadershipTestCase { + description: "SQLite contenders and takeover", + backend: Backend::Sqlite, + expected_epoch: 2, + }, + LeadershipTestCase { + description: "PostgreSQL contenders and takeover", + backend: Backend::Postgres, + expected_epoch: 2, + }, + ]; + for test_case in test_cases { + let observed = run(leadership_contract(test_case.backend)); + assert!( + observed.is_none_or(|epoch| epoch == test_case.expected_epoch), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs new file mode 100644 index 0000000000..c7db9ff190 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -0,0 +1,10 @@ +//! Leadership scenarios run independently on both supported engines. + +use crate::store_test_types::Backend; + +#[derive(Debug)] +pub(crate) struct LeadershipTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_epoch: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs new file mode 100644 index 0000000000..a554a78fc6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -0,0 +1,10 @@ +#[path = "snapshots/test_contract.rs"] +mod contract; +#[path = "snapshots/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "snapshots/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs new file mode 100644 index 0000000000..aaefbd5da1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -0,0 +1,294 @@ +//! Real control-store integration coverage; both engines exercise the same public contract. + +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::snapshots::main::current_snapshot::current_snapshot; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::SchemaRejectionCase; + +fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: true, + git_author: None, + image_id: None, + environment_hash: "synthetic-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([ + ( + "customers".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table", "columns": ["customer_id"]}), + raw_sql: Some("select 1 as customer_id".into()), + compiled_sql: Some("select 1 as customer_id".into()), + python_source: None, + }, + ), + ( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"dependencies": ["customers"], "materialization": "view"}), + raw_sql: Some("select customer_id from ref('customers')".into()), + compiled_sql: Some("select customer_id from main.customers".into()), + python_source: Some("# display only: orders".into()), + }, + ), + ]), + documents: BTreeMap::from([("lineage".into(), json!({"orders": ["customers"]}))]), + } +} + +async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { + let second = second_store(store).await; + let (first, second_migration) = tokio::join!(migrate_store(store), migrate_store(&second)); + first.expect("first migration succeeds"); + second_migration.expect("concurrent migration succeeds"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create test location"); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read empty target") + .is_none() + ); + let mut input = snapshot(location); + let initial = publish_snapshot(store, &input) + .await + .expect("publish initial snapshot"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read initial snapshot") + .expect("initial snapshot exists"); + assert_eq!(loaded.record, initial); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.documents, input.documents); + assert_eq!( + loaded + .changes + .iter() + .map(|change| (change.subject_key.as_str(), change.change_kind.as_str())) + .collect::>(), + [("customers", "added"), ("orders", "added")] + ); + + // Container insertion order and the generation do not affect the content address. + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .definition = json!({"materialization": "view", "dependencies": ["customers"]}); + let unchanged = publish_snapshot(store, &input) + .await + .expect("publish reordered definition"); + assert_eq!(unchanged.manifest_digest, initial.manifest_digest); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read unchanged snapshot") + .expect("unchanged snapshot exists") + .changes + .is_empty() + ); + assert!(matches!( + publish_snapshot(store, &input).await, + Err(ControlError::StaleGeneration) + )); + + input.generation = 3; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select customer_id + 5 from main.customers".into()); + let changed = publish_snapshot(store, &input) + .await + .expect("publish changed SQL"); + assert_ne!(changed.manifest_digest, unchanged.manifest_digest); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read changed snapshot") + .expect("changed snapshot exists"); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert_eq!(loaded.changes[0].change_kind, "changed"); + assert_ne!(loaded.changes[0].before_hash, loaded.changes[0].after_hash); + + // One target's generation does not suppress another target's first publication. + input.target = "testing".into(); + input.generation = 1; + publish_snapshot(store, &input) + .await + .expect("publish another target"); + assert_eq!( + current_snapshot(store, location, "development") + .await + .expect("read development target") + .expect("development target exists") + .record + .generation, + 3 + ); + assert_eq!( + current_snapshot(store, location, "testing") + .await + .expect("read testing target") + .expect("testing target exists") + .record + .generation, + 1 + ); + + input.target = "development".into(); + input.generation = 4; + input.nodes.remove("orders"); + publish_snapshot(store, &input) + .await + .expect("publish node removal"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read node removal") + .expect("snapshot exists after removal"); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].change_kind, "removed"); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert!(loaded.changes[0].after_hash.is_none()); + + // The newer compiler wins regardless of which publisher takes the transaction lock first. + let mut older = input.clone(); + older.generation = 5; + input.generation = 6; + input + .nodes + .get_mut("customers") + .expect("customers node exists") + .compiled_sql = Some("select 6 as customer_id".into()); + let (old_result, new_result) = tokio::join!( + publish_snapshot(store, &older), + publish_snapshot(&second, &input) + ); + assert!(old_result.is_ok() || matches!(old_result, Err(ControlError::StaleGeneration))); + new_result.expect("new generation wins"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read winning snapshot") + .expect("winning snapshot exists"); + assert_eq!(loaded.record.generation, 6); + assert_eq!(loaded.nodes, input.nodes); + close_store(second).await; + loaded.record.generation +} + +pub(super) async fn snapshot_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let generation = exercise_snapshot_contract(&test_store.store).await; + verify_sqlite_deduplication(&test_store.store); + close_test_store(test_store).await; + Some(generation) +} + +fn verify_sqlite_deduplication(store: &ControlStore) { + let ControlStore::Sqlite(path) = store else { + return; + }; + let connection = rusqlite::Connection::open(path).expect("inspect control store"); + let definitions: i64 = connection + .query_row("SELECT count(*) FROM sqb_node_definitions", [], |row| { + row.get(0) + }) + .expect("count immutable definitions"); + // Two initial nodes, one changed source, and one changed customer; unchanged targets reuse them. + assert_eq!(definitions, 4); + let sources: i64 = connection + .query_row("SELECT count(*) FROM sqb_source_texts", [], |row| { + row.get(0) + }) + .expect("count shared sources"); + assert_eq!(sources, 6); +} + +pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { + let directory = tempfile::tempdir().expect("create test directory"); + let path = directory.path().join("control.sqlite3"); + let store = open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open test store"); + migrate_store(&store).await.expect("migrate test store"); + let location = create_location(&store, "orders", "test-admin") + .await + .expect("create test location"); + let mut input = snapshot(location); + let good = publish_snapshot(&store, &input) + .await + .expect("publish good snapshot"); + let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); + // Fail after immutable bodies were inserted but before switching the current pointer. + connection.execute_batch("CREATE TRIGGER fail_snapshot_switch BEFORE UPDATE ON sqb_current_snapshots BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END").expect("install publication failure"); + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select 99".into()); + assert!(publish_snapshot(&store, &input).await.is_err()); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read preserved snapshot") + .expect("last good snapshot exists") + .record, + good + ); + let count: i64 = connection + .query_row("SELECT count(*) FROM sqb_snapshots", [], |row| row.get(0)) + .expect("count committed snapshots"); + assert_eq!(count, 1); + connection + .execute_batch("DROP TRIGGER fail_snapshot_switch") + .expect("remove injected failure"); + let recovered = publish_snapshot(&store, &input) + .await + .expect("retry publication"); + for case in [ + SchemaRejectionCase { + description: "minimum server version is newer", + update_sql: "UPDATE sqb_schema_migrations SET min_server_version='9999.0.0'", + }, + SchemaRejectionCase { + description: "migration checksum changed", + update_sql: "UPDATE sqb_schema_migrations SET min_server_version='0.0.0', checksum='different'", + }, + SchemaRejectionCase { + description: "migration version is unknown", + update_sql: "UPDATE sqb_schema_migrations SET version=2", + }, + ] { + connection + .execute_batch(case.update_sql) + .expect("apply synthetic schema incompatibility"); + assert!( + matches!(migrate_store(&store).await, Err(ControlError::Schema(_))), + "{}", + case.description + ); + } + recovered.generation +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs new file mode 100644 index 0000000000..8096938089 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -0,0 +1,48 @@ +use crate::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::{RecoveryTestCase, SnapshotTestCase}; + +#[test] +fn given_control_store_when_snapshots_publish_then_generations_sources_and_changes_are_consistent() +{ + let test_cases = [ + SnapshotTestCase { + description: "SQLite snapshot publication", + backend: Backend::Sqlite, + expected_generation: 6, + }, + SnapshotTestCase { + description: "PostgreSQL snapshot publication", + backend: Backend::Postgres, + expected_generation: 6, + }, + ]; + for test_case in test_cases { + let Some(observed) = run(snapshot_contract(test_case.backend)) else { + continue; + }; + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_when_a_publish_fails_then_last_good_snapshot_survives_and_schema_drift_is_rejected() +{ + let test_cases = [RecoveryTestCase { + description: "atomic rollback and recovery", + expected_generation: 2, + }]; + for test_case in test_cases { + let observed = run(sqlite_failed_publish_and_schema_floor()); + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs new file mode 100644 index 0000000000..cd4379f770 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -0,0 +1,22 @@ +//! Cases that alter a disposable schema to verify startup rejection. + +use crate::store_test_types::Backend; + +#[derive(Debug)] +pub(super) struct SchemaRejectionCase { + pub(super) description: &'static str, + pub(super) update_sql: &'static str, +} + +#[derive(Debug)] +pub(super) struct SnapshotTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) expected_generation: i64, +} + +#[derive(Debug)] +pub(super) struct RecoveryTestCase { + pub(super) description: &'static str, + pub(super) expected_generation: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs new file mode 100644 index 0000000000..07aafd8e98 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -0,0 +1,96 @@ +//! Real-database setup shared by storage, scheduling, and queue contracts. + +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::store_test_types::{Backend, TestStore}; + +pub(crate) fn run(future: impl std::future::Future) -> T { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create control-store test runtime") + .block_on(future) +} + +pub(crate) async fn open_test_store(backend: Backend) -> Option { + let Backend::Postgres = backend else { + let directory = tempfile::tempdir().expect("create isolated SQLite directory"); + let store = open_store( + directory + .path() + .join("control.sqlite3") + .to_str() + .expect("UTF-8 test path"), + ) + .await + .expect("open isolated SQLite store"); + return Some(TestStore { + store, + directory: Some(directory), + postgres: None, + }); + }; + let Ok(url) = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL") else { + assert!( + std::env::var_os("SQLBUILD_REQUIRE_POSTGRES_TESTS").is_none(), + "PostgreSQL test URL is required" + ); + eprintln!("PostgreSQL integration requires SQLBUILD_TEST_CONTROL_POSTGRES_URL"); + return None; + }; + let admin = sqlx_postgres::PgPool::connect(&url) + .await + .expect("connect to test PostgreSQL"); + let schema = format!("sqb_test_{}", Uuid::now_v7().simple()); + sqlx_core::query::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await + .expect("create isolated test schema"); + let separator = ['?', '&'][usize::from(url.contains('?'))]; + let store = open_store(&format!("{url}{separator}options=-csearch_path%3D{schema}")) + .await + .expect("open isolated PostgreSQL store"); + Some(TestStore { + store, + directory: None, + postgres: Some((admin, schema)), + }) +} + +pub(crate) async fn close_test_store(test_store: TestStore) { + close_store(test_store.store).await; + let Some((admin, schema)) = test_store.postgres else { + return; + }; + sqlx_core::query::query(&format!("DROP SCHEMA {schema} CASCADE")) + .execute(&admin) + .await + .expect("remove isolated test schema"); + admin.close().await; + drop(test_store.directory); +} + +pub(crate) async fn close_store(store: ControlStore) { + let ControlStore::Postgres(pool) = store else { + return; + }; + pool.close().await; +} + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + panic!("expected SQLite fixture"); + }; + return open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"); + }; + ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs new file mode 100644 index 0000000000..55a1a65092 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -0,0 +1,16 @@ +//! Store fixtures retain their isolated SQLite directory or PostgreSQL schema until cleanup. + +use sqlbuild_orchestration::store::models::ControlStore; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Backend { + Sqlite, + Postgres, +} + +#[derive(Debug)] +pub(crate) struct TestStore { + pub(crate) store: ControlStore, + pub(crate) directory: Option, + pub(crate) postgres: Option<(sqlx_postgres::PgPool, String)>, +} diff --git a/crates/sqlbuild-orchestration/tests/ticks.rs b/crates/sqlbuild-orchestration/tests/ticks.rs new file mode 100644 index 0000000000..00bfc0cbb6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks.rs @@ -0,0 +1,16 @@ +#[path = "ticks/fault_helpers.rs"] +mod fault_helpers; +#[path = "ticks/helpers.rs"] +mod helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "ticks/test_contract.rs"] +mod test_contract; +#[path = "ticks/test_loop.rs"] +mod test_loop; +#[path = "ticks/test_triggers.rs"] +mod test_triggers; +#[path = "ticks/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs new file mode 100644 index 0000000000..180b1babce --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs @@ -0,0 +1,66 @@ +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::test_types::TickFaultTestCase; + +pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCase) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(&format!( + "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", + point.operation, point.table, point.condition, point.sqlite_body + )) + .expect("install SQLite transaction fault"); + return; + }; + sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) + .execute(pool).await.expect("install PostgreSQL fault function"); + sqlx_core::query::query(&format!("CREATE TRIGGER sqb_test_fault BEFORE {} ON {} FOR EACH ROW {} EXECUTE FUNCTION sqb_test_fault()", + point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + &[")", ""][usize::from(point.condition.is_empty())])) + .execute(pool).await.expect("install PostgreSQL transaction fault"); +} + +pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch("DROP TRIGGER sqb_test_fault") + .expect("remove SQLite transaction fault"); + return; + }; + sqlx_core::query::query(&format!("DROP TRIGGER sqb_test_fault ON {}", point.table)) + .execute(pool) + .await + .expect("remove PostgreSQL fault trigger"); + sqlx_core::query::query("DROP FUNCTION sqb_test_fault()") + .execute(pool) + .await + .expect("remove PostgreSQL fault function"); +} + +pub(crate) async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL lease"); +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs new file mode 100644 index 0000000000..183b105cde --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs @@ -0,0 +1,240 @@ +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::scheduler::main::list_ticks::list_ticks; +use sqlbuild_orchestration::scheduler::models::TickOutcome; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::TickFaultTestCase; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ( + "triggers".into(), + json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + ), + ]), + } +} + +pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate tick store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish job and trigger"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("published triggers")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("elected"); + let second = second_store(store).await; + for point in [ + TickFaultTestCase { + description: "tick insertion", + table: "sqb_ticks", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected tick failure')", + postgres_body: "RAISE EXCEPTION 'injected tick failure'", + }, + TickFaultTestCase { + description: "requested event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_requested'", + sqlite_body: "SELECT RAISE(ABORT,'injected requested failure')", + postgres_body: "RAISE EXCEPTION 'injected requested failure'", + }, + TickFaultTestCase { + description: "queued event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_queued'", + sqlite_body: "SELECT RAISE(ABORT,'injected queued failure')", + postgres_body: "RAISE EXCEPTION 'injected queued failure'", + }, + TickFaultTestCase { + description: "run projection", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected run failure')", + postgres_body: "RAISE EXCEPTION 'injected run failure'", + }, + TickFaultTestCase { + description: "node projection", + table: "sqb_run_nodes", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }, + TickFaultTestCase { + description: "cursor commit", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected cursor failure'", + }, + TickFaultTestCase { + description: "lease expires during append", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }, + ] { + install_fault(store, &point).await; + let failed = evaluate_trigger(store, &lease, trigger_id).await; + assert!(failed.is_err(), "{} must abort", point.description); + remove_fault(store, &point).await; + assert!( + list_ticks(&second, trigger_id, 100) + .await + .expect("rolled-back ticks") + .is_empty(), + "{}", + point.description + ); + assert!( + list_runs(&second, (location, "development"), 100) + .await + .expect("rolled-back runs") + .is_empty(), + "{}", + point.description + ); + assert!( + read_recall_events(&second, 0, 100) + .await + .expect("rolled-back events") + .is_empty(), + "{}", + point.description + ); + assert!( + list_triggers(&second, location, "development") + .await + .expect("rolled-back cursor")[0] + .last_tick_for + .is_none(), + "{}", + point.description + ); + } + let (left, right) = tokio::join!( + evaluate_trigger(store, &lease, trigger_id), + evaluate_trigger(&second, &lease, trigger_id) + ); + let first_count = left + .expect("first scheduler") + .expect("active trigger") + .actions + .len(); + let second_count = right + .expect("second scheduler") + .expect("active trigger") + .actions + .len(); + assert_eq!( + first_count + second_count, + 3, + "restarts never repeat committed ticks" + ); + let ticks = list_ticks(store, trigger_id, 100) + .await + .expect("committed ticks"); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("committed run"); + let events = read_recall_events(store, 0, 100) + .await + .expect("committed events"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("event type")) + .collect::>(), + ["run_requested", "run_queued"] + ); + assert_eq!(ticks[0].outcome, TickOutcome::Requested); + assert!( + ticks[1..] + .iter() + .all(|tick| tick.outcome == TickOutcome::Covered + && tick.covered_by_run_id == Some(runs[0].run_id)) + ); + assert_eq!( + runs[0].last_event_storage_id, + events[1]["storage_id"].as_i64().expect("event cursor") + ); + expire_lease(store).await; + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + evaluate_trigger(store, &lease, trigger_id).await, + Err(ControlError::Fenced) + )); + assert!( + evaluate_trigger(&second, &successor, trigger_id) + .await + .expect("failover catch-up") + .expect("active trigger") + .actions + .is_empty() + ); + let counts = (ticks.len(), runs.len()); + close_store(second).await; + close_test_store(fixture).await; + Some(counts) +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs new file mode 100644 index 0000000000..7d733d45be --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs @@ -0,0 +1,32 @@ +use crate::helpers::transaction_contract; +use crate::store_helpers::run; +use crate::store_test_types::Backend; +use crate::test_types::TickTransactionTestCase; + +#[test] +fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite fenced tick transaction", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL fenced tick transaction", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + let observed = run(transaction_contract(test_case.backend)); + assert!( + observed + .is_none_or(|counts| counts == (test_case.expected_ticks, test_case.expected_runs)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs new file mode 100644 index 0000000000..34b691b8fa --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs @@ -0,0 +1,98 @@ +use std::time::Duration; + +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::fault_helpers::expire_lease; +use crate::helpers::snapshot; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::store_test_types::Backend; +use crate::test_types::TickTransactionTestCase; + +#[test] +fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite scheduler loop", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL scheduler loop", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate loop store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish due schedule"); + let second = second_store(store).await; + let mut first_state = SchedulerState::new(Uuid::now_v7()); + let mut second_state = SchedulerState::new(Uuid::now_v7()); + let first = scheduler_cycle(store, &mut first_state) + .await + .expect("initial cycle"); + assert!(first.leader, "{}", test_case.description); + assert_eq!(first.evaluated, 1); + assert_eq!(first.ticks, test_case.expected_ticks); + assert_eq!(first.wait_for, Duration::from_secs(5)); + let follower = scheduler_cycle(&second, &mut second_state) + .await + .expect("follower cycle"); + assert!(!follower.leader); + assert_eq!(follower.ticks, 0); + expire_lease(store).await; + let successor = scheduler_cycle(&second, &mut second_state) + .await + .expect("takeover cycle"); + assert!(successor.leader); + assert_eq!(successor.ticks, 0, "takeover never repeats durable work"); + assert!( + !scheduler_cycle(store, &mut first_state) + .await + .expect("fenced cycle") + .leader + ); + let runs = sqlbuild_orchestration::runs::main::list_runs::list_runs( + store, + (location, "development"), + 100, + ) + .await + .expect("queued run"); + assert_eq!(runs.len(), test_case.expected_runs); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let (_changed, changes) = tokio::sync::watch::channel(()); + let mut running = Box::pin(run_scheduler(&second, second_state, shutdown, changes)); + tokio::select! { + result = &mut running => panic!("loop ended before shutdown: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + stop.send(true).expect("signal shutdown"); + tokio::time::timeout(Duration::from_secs(2), running) + .await + .expect("idle shutdown does not wait for the five-second timer") + .expect("scheduler exits cleanly"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs new file mode 100644 index 0000000000..4429dae9fb --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs @@ -0,0 +1,195 @@ +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::declarations::models::{ + TriggerChange, TriggerChangeRequest, TriggerStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{TickFaultTestCase, TriggerControlsTestCase}; + +#[test] +fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() + { + let test_cases = [ + TriggerControlsTestCase { + description: "SQLite operator facts", + backend: Backend::Sqlite, + expected_events: 5, + }, + TriggerControlsTestCase { + description: "PostgreSQL operator facts", + backend: Backend::Postgres, + expected_events: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate trigger store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("initial publish"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let pause = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "orders-operator".into(), + reason: "maintenance".into(), + change: TriggerChange::Pause, + }; + assert!(change_trigger(store, &pause).await.expect("pause")); + let paused = list_triggers(store, location, "development") + .await + .expect("paused trigger") + .remove(0); + assert_eq!( + paused.status, + TriggerStatus::Paused, + "{}", + test_case.description + ); + assert!(paused.next_due_at.is_none()); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + assert!( + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("paused evaluation") + .is_none() + ); + let enable = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::Enable, + ..pause.clone() + }; + let fault = TickFaultTestCase { + description: "operator event and projection", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", + postgres_body: "RAISE EXCEPTION 'injected state failure'", + }; + install_fault(store, &fault).await; + assert!(change_trigger(store, &enable).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_trigger_events(store, trigger_id, 100) + .await + .expect("rolled-back fact") + .len(), + 1 + ); + assert!(change_trigger(store, &enable).await.expect("retry enable")); + assert!( + !change_trigger(store, &pause) + .await + .expect("retry old pause is inert") + ); + let conflict = TriggerChangeRequest { + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + assert!( + change_trigger(store, &conflict).await.is_err(), + "same event ID cannot change content" + ); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document")["daily_orders"]["default_status"] = + serde_json::json!("paused"); + publish_snapshot(store, &input) + .await + .expect("publish a changed code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("retained override")[0] + .status, + TriggerStatus::Running + ); + let reset = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + change_trigger(store, &reset) + .await + .expect("restore new code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("default restored")[0] + .status, + TriggerStatus::Paused + ); + let time = chrono::DateTime::parse_from_rfc3339("2020-01-02T00:00:00Z") + .expect("cursor") + .to_utc(); + let cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::SetCursor { time }, + ..pause.clone() + }; + change_trigger(store, &cursor) + .await + .expect("set durable cursor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("cursor set")[0] + .last_tick_for, + Some(time) + ); + let reset_cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetCursor, + ..pause.clone() + }; + change_trigger(store, &reset_cursor) + .await + .expect("reset cursor"); + assert!( + list_triggers(store, location, "development") + .await + .expect("cursor reset")[0] + .last_tick_for + .is_none() + ); + let events = list_trigger_events(store, trigger_id, 100) + .await + .expect("operator history"); + assert_eq!(events.len(), test_case.expected_events); + assert!( + events + .iter() + .all(|event| event.request_json.actor == "orders-operator") + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs new file mode 100644 index 0000000000..05ee9cd1d5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs @@ -0,0 +1,23 @@ +use crate::store_test_types::Backend; + +pub(crate) struct TickTransactionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, + pub(crate) expected_runs: usize, +} + +pub(crate) struct TickFaultTestCase { + pub(crate) description: &'static str, + pub(crate) table: &'static str, + pub(crate) operation: &'static str, + pub(crate) condition: &'static str, + pub(crate) sqlite_body: &'static str, + pub(crate) postgres_body: &'static str, +} + +pub(crate) struct TriggerControlsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_events: usize, +} diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index 5738b05067..cfabda69e4 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] blake3.workspace = true +chrono.workspace = true rusqlite.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 9863904c39..784b5e1cd3 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -1,6 +1,7 @@ //! ClickHouse literals and key expressions for the store's own queries. use crate::clickhouse::constants::TIMESTAMP_PATTERN; +use crate::records::_helpers::rows; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::parse_timestamp::parse_timestamp; use crate::records::main::timestamp_micros::timestamp_micros; @@ -62,10 +63,6 @@ pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) } - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), + _ => rows::key_text(value), } } diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs index 5ad2768b37..f5b60dc1e5 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -23,6 +23,17 @@ pub(crate) fn boolean(value: bool) -> RecallValue { RecallValue::Integer(i64::from(value)) } +/// The human-readable part of a conflict key, without dialect-specific timestamp normalization. +pub(crate) fn key_text(value: Option<&RecallValue>) -> String { + match value { + Some(RecallValue::Text(text)) => text.clone(), + Some(RecallValue::Integer(number)) => number.to_string(), + Some(RecallValue::Real(number)) => number.to_string(), + Some(RecallValue::Blob(_)) => "".to_owned(), + Some(RecallValue::Null) | None => String::new(), + } +} + /// `values` plus the envelope columns and `content_hash`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. pub(crate) fn finish_row( table: Table, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index 275383fa1c..e40b47cb19 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod clickhouse_views; +pub(crate) mod postgres_views; +pub(crate) mod relational_tables; +pub(crate) mod relational_views; pub(crate) mod rendering; -pub(crate) mod sqlite_views; pub(crate) mod version_row; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs new file mode 100644 index 0000000000..98515a0e19 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs @@ -0,0 +1,91 @@ +//! PostgreSQL expressions for the shared relational Recall view definitions. + +use crate::schema::_helpers::relational_views::relational_view_bodies; + +fn json_fields(mut body: String) -> String { + for (field, cast) in [ + ("audit_name", ""), + ("audit_definition_name", ""), + ("target_name", ""), + ("target_database", ""), + ("target_schema", ""), + ("attached_target_kind", ""), + ("attached_target_name", ""), + ("attached_column_name", ""), + ("severity", ""), + ("violation_count", "::BIGINT"), + ("measured_value", "::DOUBLE PRECISION"), + ("evaluation_mode", ""), + ("execution_error", ""), + ("evidence_count", "::BIGINT"), + ("evidence_truncated", "::BOOLEAN::INTEGER"), + ("sample_digest", ""), + ("error_message", ""), + ("signal_number", "::BIGINT"), + ("cost_status", ""), + ] { + body = body.replace( + &format!("json_extract(payload, '$.{field}')"), + &format!("(payload ->> '{field}'){cast}"), + ); + } + for field in ["name", "job_name", "run_id", "retry_number"] { + let cast = if field == "retry_number" { + "::BIGINT" + } else { + "" + }; + body = body.replace( + &format!("json_extract(context.external_context, '$.integration.{field}')"), + &format!("(context.external_context #>> '{{integration,{field}}}'){cast}"), + ); + } + body.replace( + "json_valid(context.external_context)", + "context.external_context IS NOT NULL", + ) + .replace( + "json_array_length(context.selection)", + "jsonb_array_length(context.selection)", + ) +} + +fn postgres_body(body: String) -> String { + let mut body = json_fields(body) + .replace("strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", "CURRENT_TIMESTAMP") + .replace("MAX(artifact.digest IS NOT NULL)", "MAX(CAST(artifact.digest IS NOT NULL AS INTEGER))") + .replace("artifact.digest IS NOT NULL AS available", "CAST(artifact.digest IS NOT NULL AS INTEGER) AS available") + .replace("MAX(\n COALESCE(received.max_sequence", "GREATEST(\n COALESCE(received.max_sequence") + .replace("MAX(COALESCE(received.queue_drops", "GREATEST(COALESCE(received.queue_drops") + .replace("COALESCE(context.recall_filter_active, 0)", "COALESCE(CAST(context.recall_filter_active AS INTEGER), 0)") + .replace("COALESCE(delivery.flush_complete, 0)", "COALESCE(CAST(delivery.flush_complete AS INTEGER), 0)") + // PostgreSQL text cannot hold NUL or malformed UTF-8. Keep display bytes until the reader decodes them. + .replace("CAST(data AS TEXT)", "data") + .replace("CAST(artifact.content AS TEXT)", "artifact.content") + .replace(")\nGROUP BY", ") AS source_rows\nGROUP BY") + .replace(")\n GROUP BY", ") AS source_rows\n GROUP BY") + .replace(")\nWHERE position", ") AS source_rows\nWHERE position") + .replace("ROUND(\n (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3\n )", + "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)"); + for expression in [ + "byte_count", + "CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END", + "CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END", + "dropped_before", + "COALESCE(omitted_sequences, 1)", + "COALESCE(omitted_bytes, 0)", + ] { + body = body.replace( + &format!("SUM({expression})"), + &format!("CAST(SUM({expression}) AS BIGINT)"), + ); + } + body +} + +pub(crate) fn postgres_view_bodies() -> Vec<(&'static str, String)> { + relational_view_bodies(true) + .into_iter() + .map(|(name, body)| (name, postgres_body(body))) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs new file mode 100644 index 0000000000..3930030bd3 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -0,0 +1,69 @@ +//! One logical schema and index list for SQLite and PostgreSQL. + +use crate::schema::_helpers::rendering::{all_columns, sqlite_identifier, sqlite_type}; +use crate::schema::models::{ColumnType, TableSpec}; + +pub(crate) fn postgres_type(column: ColumnType) -> &'static str { + match column { + ColumnType::Text => "TEXT", + ColumnType::Timestamp => "TIMESTAMPTZ", + ColumnType::Json => "JSONB", + ColumnType::Integer => "BIGINT", + ColumnType::Boolean => "BOOLEAN", + ColumnType::Real => "DOUBLE PRECISION", + ColumnType::Blob => "BYTEA", + } +} + +pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { + let mut definitions: Vec = all_columns(spec) + .iter() + .map(|column| { + let not_null = if column.nullable { "" } else { " NOT NULL" }; + let kind = if postgres { + postgres_type(column.column_type) + } else if control && column.column_type == ColumnType::Json { + "JSON" + } else { + sqlite_type(column.column_type) + }; + format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) + }) + .collect(); + if control && spec.name == "recall_events" { + definitions.push(if postgres { + " storage_id BIGINT GENERATED ALWAYS AS IDENTITY".into() + } else { + " storage_id INTEGER PRIMARY KEY AUTOINCREMENT".into() + }); + } + let mut key: Vec<&str> = spec.key.to_vec(); + let partition = if postgres && spec.name != "recall_artifacts" { + if !key.contains(&spec.expiry_column) { + key.push(spec.expiry_column); + } + format!( + " PARTITION BY RANGE ({})", + sqlite_identifier(spec.expiry_column) + ) + } else { + String::new() + }; + let key: Vec = key.into_iter().map(sqlite_identifier).collect(); + definitions.push(format!(" UNIQUE ({})", key.join(", "))); + let mut statements = vec![format!( + "CREATE TABLE IF NOT EXISTS {} (\n{}\n){partition}", + sqlite_identifier(spec.name), + definitions.join(",\n") + )]; + for (suffix, columns) in spec.sqlite_indexes { + let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); + statements.push(format!( + "CREATE INDEX IF NOT EXISTS {} ON {} ({})", + sqlite_identifier(&format!("{}_{suffix}", spec.name)), + sqlite_identifier(spec.name), + columns.join(", ") + )); + } + statements +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs similarity index 92% rename from crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs rename to crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 8d4e1c2f81..015c491b5e 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,4 +1,4 @@ -//! The bodies of the supported views in SQLite. +//! Shared relational view definitions, with explicit dialect expressions for binary transcripts. use crate::schema::constants::OUTPUT_TAIL_RECORDS; @@ -212,18 +212,29 @@ SELECT END AS complete FROM counted"; -const OUTPUT_TRANSCRIPTS: &str = "SELECT - invocation_id, - MIN(invocation_started_at) AS invocation_started_at, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stdout, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stderr, - CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined -FROM recall_output -GROUP BY invocation_id"; +fn output_transcripts(postgres: bool) -> String { + let columns: Vec = [ + ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), + ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), + ("data", "combined"), + ] + .into_iter() + .map(|(expression, alias)| { + let value = if postgres { + format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") + } else { + format!( + "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" + ) + }; + format!(" {value} AS {alias}") + }) + .collect(); + format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", + columns.join(",\n") + ) +} const EVENTS: &str = "SELECT * FROM recall_events"; @@ -414,15 +425,15 @@ WHERE position <= {OUTPUT_TAIL_RECORDS}" ) } -/// Each supported view's name and SQLite body, in creation order. -pub(crate) fn sqlite_view_bodies() -> Vec<(&'static str, String)> { +/// Each supported view's name and relational body, in creation order. +pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { vec![ ("invocations", invocations()), ("statements", STATEMENTS.to_owned()), ("audit_results", AUDIT_RESULTS.to_owned()), ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), ("output_tail", output_tail()), - ("output_transcripts", OUTPUT_TRANSCRIPTS.to_owned()), + ("output_transcripts", output_transcripts(postgres)), ("events", EVENTS.to_owned()), ("diagnostics", DIAGNOSTICS.to_owned()), ("artifacts", ARTIFACTS.to_owned()), diff --git a/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs new file mode 100644 index 0000000000..db06b995c6 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs @@ -0,0 +1,14 @@ +//! Control stores preserve Recall's logical columns and add a committed-order event cursor. + +use crate::schema::_helpers::relational_tables::relational_table; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; + +/// PostgreSQL partitions invocation facts; globally deduplicated artifacts remain unpartitioned. +#[must_use] +pub fn control_table_statements(postgres: bool) -> Vec { + ALL_TABLES + .iter() + .flat_map(|table| relational_table(table_spec(*table), postgres, true)) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs new file mode 100644 index 0000000000..7b27ec1a2f --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -0,0 +1,34 @@ +//! Control stores expose Recall's supported views on both engines. + +use crate::schema::_helpers::postgres_views::postgres_view_bodies; +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// PostgreSQL validates each view at migration time; SQLite retains its existing view contract. +#[must_use] +pub fn control_view_statements(postgres: bool) -> Vec { + if postgres { + postgres_view_bodies() + .into_iter() + .map(|(name, body)| { + format!( + "CREATE OR REPLACE VIEW {} AS\n{body}", + sqlite_identifier(name) + ) + }) + .collect() + } else { + view_statements( + Dialect::Sqlite, + &DdlOptions { + database: None, + retention_days: 30, + cluster: None, + }, + ) + .into_iter() + .map(|(_, statement)| statement) + .collect() + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index e2251bd838..879cacea23 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -1,8 +1,11 @@ pub mod add_column_statement; pub mod clickhouse_relation; +pub mod control_table_statements; +pub mod control_view_statements; pub mod create_database_statement; pub mod ddl_script; pub mod ddl_statements; +pub mod postgres_partition_statement; pub mod render_script; pub mod retention_fix_statements; pub mod schema_age; diff --git a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs new file mode 100644 index 0000000000..971a3ae792 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs @@ -0,0 +1,24 @@ +//! Monthly partition bounds are calculated from a typed date, never interpolated user SQL. + +use chrono::{Datelike, Months, NaiveDate}; + +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; + +/// A replay uses the same timestamp partition and therefore the same deterministic unique key. +#[must_use] +pub fn postgres_partition_statement(table: Table, date: NaiveDate) -> Option { + if table == Table::Artifacts { + return None; + } + let first = date.with_day(1)?; + let next = first.checked_add_months(Months::new(1))?; + let spec = table_spec(table); + let partition = format!("{}_{}", spec.name, first.format("%Y%m")); + Some(format!( + "CREATE TABLE IF NOT EXISTS {} PARTITION OF {} FOR VALUES FROM ('{first}T00:00:00Z') TO ('{next}T00:00:00Z')", + sqlite_identifier(&partition), + sqlite_identifier(spec.name) + )) +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_statements.rs b/crates/sqlbuild-recall/src/schema/main/table_statements.rs index 621f8e1f12..7829997b9b 100644 --- a/crates/sqlbuild-recall/src/schema/main/table_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/table_statements.rs @@ -1,48 +1,14 @@ //! `table_statements`: idempotent `CREATE TABLE` and index statements of every Recall table. +use crate::schema::_helpers::relational_tables::relational_table; use crate::schema::_helpers::rendering::{ all_columns, clickhouse_engine, clickhouse_identifier, clickhouse_table, clickhouse_ttl, - clickhouse_type, on_cluster, sqlite_identifier, sqlite_type, + clickhouse_type, on_cluster, }; use crate::schema::constants::ALL_TABLES; use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Dialect, TableSpec}; -fn sqlite_table(spec: &TableSpec) -> Vec { - let mut definitions: Vec = all_columns(spec) - .iter() - .map(|column| { - let not_null: &str = if column.nullable { "" } else { " NOT NULL" }; - format!( - " {} {}{not_null}", - sqlite_identifier(column.name), - sqlite_type(column.column_type) - ) - }) - .collect(); - let key: Vec = spec - .key - .iter() - .map(|name| sqlite_identifier(name)) - .collect(); - definitions.push(format!(" UNIQUE ({})", key.join(", "))); - let mut statements: Vec = vec![format!( - "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", - sqlite_identifier(spec.name), - definitions.join(",\n") - )]; - for (suffix, columns) in spec.sqlite_indexes { - let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); - statements.push(format!( - "CREATE INDEX IF NOT EXISTS {} ON {} ({})", - sqlite_identifier(&format!("{}_{suffix}", spec.name)), - sqlite_identifier(spec.name), - columns.join(", ") - )); - } - statements -} - fn clickhouse_table_statement(spec: &TableSpec, options: &DdlOptions) -> String { let columns: Vec = all_columns(spec) .iter() @@ -76,7 +42,7 @@ pub fn table_statements(dialect: Dialect, options: &DdlOptions) -> Vec { .flat_map(|table| { let spec: &TableSpec = table_spec(*table); match dialect { - Dialect::Sqlite => sqlite_table(spec), + Dialect::Sqlite => relational_table(spec, false, false), Dialect::ClickHouse => vec![clickhouse_table_statement(spec, options)], } }) diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index ce34504a3c..8527aae75c 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -1,15 +1,15 @@ //! `view_statements`: the statements that create the supported views. use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; +use crate::schema::_helpers::relational_views::relational_view_bodies; use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; -use crate::schema::_helpers::sqlite_views::sqlite_view_bodies; use crate::schema::models::{DdlOptions, Dialect}; /// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. #[must_use] pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { match dialect { - Dialect::Sqlite => sqlite_view_bodies() + Dialect::Sqlite => relational_view_bodies(false) .into_iter() .map(|(name, body)| { ( diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index 3344c9a183..fab996484f 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod column_types; pub(crate) mod identity; pub(crate) mod quarantine; +pub(crate) mod relational; pub(crate) mod rows; pub(crate) mod schema; pub(crate) mod sqlite; diff --git a/crates/sqlbuild-recall/src/store/_helpers/relational.rs b/crates/sqlbuild-recall/src/store/_helpers/relational.rs new file mode 100644 index 0000000000..becce06752 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/relational.rs @@ -0,0 +1,12 @@ +//! Typed placeholders shared by native SQLite and control-plane transactions. + +use crate::schema::_helpers::relational_tables::postgres_type; +use crate::schema::models::ColumnType; + +pub(crate) fn placeholder(index: usize, kind: ColumnType, postgres: bool) -> String { + if postgres { + format!("CAST(${index} AS {})", postgres_type(kind)) + } else { + format!("${index}") + } +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index 5fed0c3900..a253f2ce78 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -5,6 +5,7 @@ use std::collections::{HashMap, HashSet}; use rusqlite::{Connection, OptionalExtension}; +use crate::records::_helpers::rows::key_text; use crate::records::main::conflict_row::conflict_row; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::main::table_spec::table_spec; @@ -12,18 +13,11 @@ use crate::schema::models::Table; use crate::store::_helpers::schema::stored_columns; use crate::store::_helpers::sqlite::{SqlValue, checked}; use crate::store::errors::StoreError; +use crate::store::main::conflict_evidence::conflict_evidence; +use crate::store::main::relational_insert::relational_insert; +use crate::store::main::relational_lookup::relational_lookup; use crate::store::models::{ConflictEvidence, InsertOutcome}; -fn key_text(value: Option<&RecallValue>) -> String { - match value { - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), - } -} - fn text_of(row: &RecallRow, column: &str) -> String { key_text(row.value(column)) } @@ -34,28 +28,17 @@ fn write_row( (row, columns): (&RecallRow, &HashSet), ingested_at: &str, ) -> Result, StoreError> { - let spec = table_spec(row.table); let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); - let stored: Vec<&(&str, RecallValue)> = row + let insert = relational_insert((row, columns), &ingested, false); + let parameters: Vec> = insert .values .iter() - .filter(|(name, _)| columns.contains(*name)) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut names: Vec<&str> = stored.iter().map(|(name, _)| *name).collect(); - names.push("ingested_at"); - let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); - let sql = format!( - "INSERT INTO \"{}\" ({}) VALUES ({}) ON CONFLICT DO NOTHING", - spec.name, - names.join(", "), - placeholders.join(", ") - ); - let parameters: Vec> = stored - .iter() - .map(|(_, value)| SqlValue(value)) - .chain(std::iter::once(SqlValue(&ingested))) - .collect(); - let mut statement = checked(connection.prepare_cached(&sql), "prepare Recall insert")?; + let mut statement = checked( + connection.prepare_cached(&insert.sql), + "prepare Recall insert", + )?; let changed: usize = checked( statement.execute(rusqlite::params_from_iter(parameters.iter())), "write Recall rows", @@ -63,24 +46,16 @@ fn write_row( if changed > 0 { return Ok(None); } - let condition: Vec = spec - .key - .iter() - .enumerate() - .map(|(index, name)| format!("{name} = ?{}", index + 1)) - .collect(); - let lookup = format!( - "SELECT content_hash FROM \"{}\" WHERE {}", - spec.name, - condition.join(" AND ") - ); - let null = RecallValue::Null; - let key_values: Vec> = spec - .key + let lookup = relational_lookup(row, false, false); + let key_values: Vec> = lookup + .values .iter() - .map(|name| SqlValue(row.value(name).unwrap_or(&null))) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut statement = checked(connection.prepare_cached(&lookup), "prepare Recall lookup")?; + let mut statement = checked( + connection.prepare_cached(&lookup.sql), + "prepare Recall lookup", + )?; let existing: Option = checked( statement .query_row(rusqlite::params_from_iter(key_values.iter()), |found| { @@ -89,21 +64,7 @@ fn write_row( .optional(), "read stored Recall row", )?; - let rejected: &str = row.content_hash().unwrap_or_default(); - match existing { - Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { - table: row.table, - record_key: spec - .key - .iter() - .map(|name| key_text(row.value(name))) - .collect::>() - .join(":"), - existing_sha256: existing, - rejected_sha256: rejected.to_owned(), - })), - _ => Ok(None), - } + Ok(existing.and_then(|existing| conflict_evidence(row, &existing))) } /// Write `rows` on the current transaction; conflicts become `recall_conflicts` rows. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs new file mode 100644 index 0000000000..a343086c2e --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -0,0 +1,23 @@ +//! Duplicate rows preserve the first content and expose the rejected digest. + +use crate::records::_helpers::rows::key_text; +use crate::records::models::RecallRow; +use crate::schema::main::table_spec::table_spec; +use crate::store::models::ConflictEvidence; + +/// Evidence only when a stored row's content differs, independent of the storage driver. +#[must_use] +pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option { + let rejected = row.content_hash().unwrap_or_default(); + (existing != rejected).then(|| ConflictEvidence { + table: row.table, + record_key: table_spec(row.table) + .key + .iter() + .map(|name| key_text(row.value(name))) + .collect::>() + .join(":"), + existing_sha256: existing.to_owned(), + rejected_sha256: rejected.to_owned(), + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 69ccb1cf7e..78f9b89028 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -1,5 +1,6 @@ pub mod checkpoint_sqlite_store; pub mod chosen_invocation; +pub mod conflict_evidence; pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; @@ -10,5 +11,7 @@ pub mod open_sqlite_store; pub mod quarantine_store; pub mod query_store; pub mod read_output_transcript; +pub mod relational_insert; +pub mod relational_lookup; pub mod resolve_invocation; pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/relational_insert.rs b/crates/sqlbuild-recall/src/store/main/relational_insert.rs new file mode 100644 index 0000000000..ef63916e65 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_insert.rs @@ -0,0 +1,48 @@ +//! One insert contract for standalone SQLite Recall and transactional control stores. + +use std::collections::HashSet; + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::ColumnType; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Ignore fields absent from an older additive schema; ingestion time is assigned by the store. +#[must_use] +pub fn relational_insert<'a>( + (row, columns): (&'a RecallRow, &HashSet), + ingested_at: &'a RecallValue, + postgres: bool, +) -> RecallStatement<'a> { + let metadata = table_columns(row.table); + let mut names = Vec::new(); + let mut values = Vec::new(); + for (name, value) in &row.values { + if let Some(column) = metadata.iter().find(|column| column.name == *name) + && columns.contains(*name) + && *name != "ingested_at" + { + names.push(sqlite_identifier(name)); + values.push((value, column.column_type)); + } + } + names.push("ingested_at".into()); + values.push((ingested_at, ColumnType::Timestamp)); + let placeholders: Vec = values + .iter() + .enumerate() + .map(|(index, (_, kind))| placeholder(index + 1, *kind, postgres)) + .collect(); + RecallStatement { + sql: format!( + "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT DO NOTHING", + sqlite_identifier(table_spec(row.table).name), + names.join(", "), + placeholders.join(", ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs new file mode 100644 index 0000000000..d1291fdb4c --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs @@ -0,0 +1,48 @@ +//! Duplicate detection uses the same stable keys as physical unique constraints. + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Control event lookups return their durable cursor too; standalone stores have no cursor column. +#[must_use] +pub fn relational_lookup(row: &RecallRow, postgres: bool, control: bool) -> RecallStatement<'_> { + let spec = table_spec(row.table); + let mut keys = spec.key.to_vec(); + if postgres && row.table != Table::Artifacts && !keys.contains(&spec.expiry_column) { + keys.push(spec.expiry_column); + } + let metadata = table_columns(row.table); + let mut values = Vec::new(); + let mut conditions = Vec::new(); + for key in keys { + let kind = metadata + .iter() + .find(|column| column.name == key) + .expect("Recall key columns belong to their schema") + .column_type; + values.push((row.value(key).unwrap_or(&RecallValue::Null), kind)); + conditions.push(format!( + "{} = {}", + sqlite_identifier(key), + placeholder(values.len(), kind, postgres) + )); + } + let cursor = if control && row.table == Table::Events { + ", storage_id" + } else { + "" + }; + RecallStatement { + sql: format!( + "SELECT content_hash{cursor} FROM {} WHERE {}", + sqlite_identifier(spec.name), + conditions.join(" AND ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index a92d4ead30..61b2f27614 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -4,7 +4,14 @@ use std::path::PathBuf; use std::time::{Duration, SystemTime}; use crate::records::models::{RecallValue, RowEnvelope}; -use crate::schema::models::Table; +use crate::schema::models::{ColumnType, Table}; + +/// A relational insert or lookup and its borrowed, typed parameters. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallStatement<'a> { + pub sql: String, + pub values: Vec<(&'a RecallValue, ColumnType)>, +} /// One open store file. Only one thread uses a store. #[derive(Debug)] diff --git a/fensu.toml b/fensu.toml index 7e2ae7561a..2f97466395 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1617,6 +1617,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python", From bbd50946b4352785f95eb0f96ab875ee20b5385e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:11:15 +0000 Subject: [PATCH 002/138] feat: checkpoint pinned worker execution verification Add claim-scoped execution inputs and native snapshot drift validation. Incomplete orchestration backend checkpoint; worker launch and the remaining control-plane integration are not implemented yet. Targeted SQLite/PostgreSQL tests pass; full static readiness is pending. --- .../src/api/_helpers/runs.rs | 21 ++- .../src/api/main/serve_api.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 8 +- .../src/runs/main/mod.rs | 1 + .../src/runs/main/request_run.rs | 6 +- .../src/runs/main/run_execution.rs | 58 ++++++ .../sqlbuild-orchestration/src/runs/models.rs | 24 ++- .../src/scheduler/models.rs | 2 +- .../src/snapshots/_helpers/content.rs | 28 ++- .../src/snapshots/main/mod.rs | 1 + .../src/snapshots/main/verify_execution.rs | 50 ++++++ .../src/snapshots/models.rs | 7 + .../tests/api/test_contract.rs | 170 +++++++++++++++++- .../tests/api/test_types.rs | 9 + 14 files changed, 363 insertions(+), 23 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/run_execution.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index 419ec2681a..ef93a4a15a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -10,9 +10,28 @@ use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::runs::main::change_run::change_run; use crate::runs::main::heartbeat_run::heartbeat_run; -use crate::runs::models::{RunAuthority, RunChangeRequest}; +use crate::runs::main::run_execution::run_execution; +use crate::runs::models::{RunAuthority, RunChangeRequest, RunExecution}; use crate::store::models::ControlStore; +#[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), + responses((status = 200, body = RunExecution), (status = 401, description = "Inactive claim or invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn execution( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + let execution = run_execution(&store, (run_id, claim_id)) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::UNAUTHORIZED, + message: "run claim is no longer active".into(), + })?; + Ok(Json(execution)) +} + #[utoipa::path(post, path = "/api/runs/{run_id}/events", params(("run_id" = Uuid, Path)), request_body = RunChangeRequest, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 51d3b48cf4..99927066e8 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -22,6 +22,7 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), ) + .route("/api/runs/{run_id}/execution", get(runs::execution)) .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 3c3590a1ba..e7db400f75 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -5,7 +5,13 @@ use serde::{Deserialize, Serialize}; use crate::api::_helpers::{runs, snapshots}; #[derive(Debug, utoipa::OpenApi)] -#[openapi(paths(snapshots::publish, snapshots::current, runs::report, runs::heartbeat))] +#[openapi(paths( + snapshots::publish, + snapshots::current, + runs::execution, + runs::report, + runs::heartbeat +))] pub(crate) struct ControlApi; #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index e3e4c9a4d4..c19b71681a 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -5,3 +5,4 @@ pub mod list_run_nodes; pub mod list_runs; pub(crate) mod queue_run; pub mod request_run; +pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs index eb55625e5d..82616a6ee2 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -41,11 +41,13 @@ pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result Result Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT r.request_json, s.environment_hash, s.sqb_version FROM sqb_runs r \ + JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND l.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let metadata: ExecutionMetadata = decode_row(row)?; + let (transaction, rows) = query( + transaction, + "SELECT node_key, node_hash FROM sqb_snapshot_nodes WHERE snapshot_id=$1", + vec![Parameter::Id(metadata.request_json.snapshot_id)], + ) + .await?; + let node_hashes: BTreeMap = rows + .into_iter() + .map(|row| { + let fields: BTreeMap = decode_row(row)?; + Ok((fields["node_key"].clone(), fields["node_hash"].clone())) + }) + .collect::>()?; + commit(transaction).await?; + Ok(Some(RunExecution { + claim: RunClaim { + claim_id, + request: metadata.request_json, + }, + environment_hash: metadata.environment_hash, + sqb_version: metadata.sqb_version, + node_hashes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 270381a7f4..636dc6891a 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunKind { Job, @@ -25,7 +25,7 @@ impl RunKind { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunOrigin { Manual, @@ -64,7 +64,7 @@ pub enum RunStatus { } /// The authorized selection and pinned execution inputs are immutable once requested. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRequest { pub run_id: Uuid, pub location_id: Uuid, @@ -141,12 +141,28 @@ pub struct RunNodeRecord { } /// The database owns the claim before a launcher sees the pinned request. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunClaim { pub claim_id: Uuid, pub request: RunRequest, } +/// Workers compile locally and compare against this immutable snapshot, not the current deploy. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunExecution { + pub claim: RunClaim, + pub environment_hash: String, + pub sqb_version: String, + pub node_hashes: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ExecutionMetadata { + pub(crate) request_json: RunRequest, + pub(crate) environment_hash: String, + pub(crate) sqb_version: String, +} + /// Terminal outcomes cannot be confused with a request to change an active status. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 7c3dbfde37..a3a0c2ccad 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -55,7 +55,7 @@ pub enum CatchUp { All { max_runs: std::num::NonZeroUsize }, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum Overlap { #[default] diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs index 4ff3f7b1f3..3ce8bbd7c3 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -7,7 +7,7 @@ use flate2::{Compression, read::GzDecoder, write::GzEncoder}; use serde_json::Value; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use crate::snapshots::models::{PreparedNode, SnapshotNode}; +use crate::snapshots::models::{NodeContent, PreparedNode, SnapshotNode}; use crate::store::errors::ControlError; fn canonical(value: &Value) -> Value { @@ -42,18 +42,15 @@ pub(crate) fn decompress(value: &Value) -> Result, ControlError> { Ok(output) } -pub(crate) fn prepare_node(node: &SnapshotNode) -> Result { - let mut sources: BTreeMap<&'static str, (String, Vec)> = BTreeMap::new(); +pub(crate) fn node_content(node: &SnapshotNode) -> Result, ControlError> { + let mut sources: BTreeMap<&'static str, (String, &str)> = BTreeMap::new(); for (kind, text) in [ ("raw_sql", &node.raw_sql), ("compiled_sql", &node.compiled_sql), ("python_source", &node.python_source), ] { if let Some(text) = text { - sources.insert( - kind, - (sha256_hex(text.as_bytes()), compress(text.as_bytes())?), - ); + sources.insert(kind, (sha256_hex(text.as_bytes()), text)); } } let source_hashes: BTreeMap<_, _> = sources @@ -63,10 +60,23 @@ pub(crate) fn prepare_node(node: &SnapshotNode) -> Result Result { + let content = node_content(node)?; + let sources = content + .sources + .into_iter() + .map(|(kind, (hash, text))| Ok((kind, (hash, compress(text.as_bytes())?)))) + .collect::>()?; Ok(PreparedNode { kind: node.kind.clone(), - hash: sha256_hex(&definition), - definition: compress(&definition)?, + hash: sha256_hex(&content.definition), + definition: compress(&content.definition)?, sources, }) } diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index da2b1bcca6..88c2bf1eba 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,2 +1,3 @@ pub mod current_snapshot; pub mod publish_snapshot; +pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs new file mode 100644 index 0000000000..d4d2c00236 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs @@ -0,0 +1,50 @@ +//! Refuse execution when a worker's local compilation differs from the pinned publication. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::runs::models::RunExecution; +use crate::snapshots::_helpers::content::node_content; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Display documents and publication generations are not execution inputs. +pub fn verify_execution( + input: &SnapshotInput, + expected: &RunExecution, +) -> Result<(), ControlError> { + let request = &expected.claim.request; + if input.location_id != request.location_id || input.target != request.target { + return Err(ControlError::Invalid( + "worker location or target differs from the pinned run".into(), + )); + } + if input.image_id != request.image_id { + return Err(ControlError::Invalid( + "worker image differs from the pinned run".into(), + )); + } + if input.environment_hash != expected.environment_hash { + return Err(ControlError::Invalid( + "worker environment differs from the pinned snapshot".into(), + )); + } + if input.sqb_version != expected.sqb_version { + return Err(ControlError::Invalid( + "worker SQLBuild version differs from the pinned snapshot".into(), + )); + } + if input.nodes.len() != expected.node_hashes.len() { + return Err(ControlError::Invalid( + "worker node set differs from the pinned snapshot".into(), + )); + } + for (key, node) in &input.nodes { + let hash = sha256_hex(&node_content(node)?.definition); + if expected.node_hashes.get(key) != Some(&hash) { + return Err(ControlError::Invalid( + "worker node content differs from the pinned snapshot".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 6f1e5da193..20d474c674 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -62,6 +62,13 @@ pub struct CurrentSnapshot { pub changes: Vec, } +/// Canonical identity is shared by publication and worker verification; only publication compresses. +#[derive(Debug)] +pub(crate) struct NodeContent<'a> { + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, &'a str)>, +} + #[derive(Debug)] pub(crate) struct PreparedNode { pub(crate) kind: String, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index ef7d0d0902..53d9903ae5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -10,9 +10,13 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope} use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunStatus}; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, +}; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; @@ -21,13 +25,26 @@ use crate::helpers::{start_api, stop_api}; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::test_types::ApiContractTestCase; #[test] fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker and location HTTP contract", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + ApiContractTestCase { + description: "PostgreSQL worker and location HTTP contract", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -88,6 +105,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie ); assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); + assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); let input = snapshot(location_id); let response = client .post(&path) @@ -132,6 +150,18 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie published.snapshot_id.to_string() ); assert_eq!(current["nodes"]["orders"]["compiled_sql"], "select 1 as id"); + for image_id in [None, Some("sha256:unpublished-orders-image".into())] { + let mut mismatched = + request((location_id, published.snapshot_id), 1, 0, &["orders"]); + mismatched.image_id = image_id; + assert!(request_run(store, &mismatched).await.is_err()); + } + assert!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("rejected requests do not queue") + .is_empty() + ); queue( store, &request((location_id, published.snapshot_id), 1, 0, &["orders"]), @@ -159,6 +189,109 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .expect("worker credential"); let events_path = format!("{}/api/runs/{run_id}/events", api.url); let heartbeat_path = format!("{}/api/runs/{run_id}/heartbeat", api.url); + let execution_path = format!("{}/api/runs/{run_id}/execution", api.url); + let response = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker execution contract"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let execution: RunExecution = response.json().await.expect("pinned execution"); + assert_eq!(execution.claim, claim); + assert_eq!(execution.environment_hash, input.environment_hash); + assert_eq!(execution.sqb_version, input.sqb_version); + assert_eq!( + execution.node_hashes.keys().collect::>(), + input.nodes.keys().collect::>(), + "dependency verification includes nodes outside the run selection" + ); + verify_execution(&input, &execution).expect("matching worker compilation"); + for field in ["target", "image_id", "environment_hash", "sqb_version"] { + let mut changed = json!(input); + changed[field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed worker inputs"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + for field in [ + "kind", + "definition", + "raw_sql", + "compiled_sql", + "python_source", + ] { + let mut changed = json!(input); + changed["nodes"]["inventory"][field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed dependency"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + let mut renamed = input.clone(); + let node = renamed.nodes.remove("inventory").expect("dependency"); + renamed.nodes.insert("renamed_inventory".into(), node); + assert!(verify_execution(&renamed, &execution).is_err()); + renamed.nodes.remove("renamed_inventory"); + assert!(verify_execution(&renamed, &execution).is_err()); + let mut display_only = input.clone(); + display_only.generation += 1; + display_only.git_author = Some("Example Author".into()); + display_only.git_dirty = true; + display_only + .documents + .insert("columns".into(), json!({"orders": ["id"]})); + verify_execution(&display_only, &execution) + .expect("display metadata does not change executable code"); + for (url, token) in [ + (execution_path.clone(), &location_token.bearer), + ( + format!("{}/api/runs/{}/execution", api.url, Uuid::now_v7()), + &worker.bearer, + ), + ] { + assert_eq!( + client + .get(url) + .bearer_auth(token) + .send() + .await + .expect("execution scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + let mut redeploy = input.clone(); + redeploy.generation += 1; + redeploy.environment_hash = "updated-orders-environment".into(); + redeploy.image_id = Some("sha256:updated-orders-image".into()); + redeploy.sqb_version = "99.0.0".into(); + redeploy + .nodes + .get_mut("orders") + .expect("orders") + .compiled_sql = Some("select 2 as id".into()); + assert_eq!( + client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish redeploy") + .status(), + StatusCode::OK + ); + let still_pinned: RunExecution = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("execution after redeploy") + .json() + .await + .expect("old execution contract"); + assert_eq!(still_pinned, execution); let started = RunChangeRequest { event_id: Uuid::now_v7(), run_id, @@ -304,13 +437,40 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .status(), StatusCode::UNAUTHORIZED ); + assert_eq!( + client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal execution credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); assert_eq!( list_runs(store, (location_id, "development"), 100) .await .expect("run projection")[0] .status, - RunStatus::Cancelled + test_case.expected_status, + "{}", + test_case.description ); + redeploy.generation += 1; + redeploy.image_id = None; + let local: SnapshotRecord = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish local snapshot without a container image") + .json() + .await + .expect("local snapshot response"); + let mut local_request = request((location_id, local.snapshot_id), 2, 0, &["orders"]); + local_request.image_id = None; + queue(store, &local_request).await; revoke_token(store, &audit, location_token.identity.token_id) .await .expect("revoke location token"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index d62cdc25ce..c3231b7060 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -1,5 +1,14 @@ +use sqlbuild_orchestration::runs::models::RunStatus; use sqlbuild_orchestration::store::errors::ControlError; +use crate::store_test_types::Backend; + +pub(crate) struct ApiContractTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} + pub(crate) struct TestApi { pub(crate) url: String, pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, From dbcd854cb71956bf469203d27e841a83089894c0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:27:25 +0000 Subject: [PATCH 003/138] feat: checkpoint authenticated worker lifecycle client Verify pinned inputs before start and expose retryable run reports and cancellation heartbeats over the authenticated API. Disable redirects for credentialed requests and cover the lifecycle against SQLite and PostgreSQL. Keep the orchestration PR explicitly incomplete pending actual launchers and the remaining backend scope. --- crates/sqlbuild-orchestration/Cargo.toml | 2 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/scheduler/_helpers/expansion.rs | 29 +- .../src/scheduler/models.rs | 3 + .../src/worker/_helpers/client.rs | 49 +++ .../src/worker/_helpers/mod.rs | 2 + .../src/worker/_helpers/response.rs | 15 + .../src/worker/errors.rs | 15 + .../src/worker/main/connect_worker.rs | 54 +++ .../src/worker/main/mod.rs | 2 + .../src/worker/main/start_worker.rs | 28 ++ .../sqlbuild-orchestration/src/worker/mod.rs | 4 + .../src/worker/models.rs | 11 + crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/test_types.rs | 5 + .../tests/api/test_worker.rs | 356 ++++++++++++++++++ 16 files changed, 562 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/client.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/response.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/errors.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/main/start_worker.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/models.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_worker.rs diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 7e9e614a65..4e31942af6 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -14,6 +14,7 @@ chrono-tz.workspace = true croner.workspace = true flate2.workspace = true getrandom.workspace = true +reqwest.workspace = true rusqlite.workspace = true semver.workspace = true serde.workspace = true @@ -29,7 +30,6 @@ uuid.workspace = true [dev-dependencies] proptest.workspace = true -reqwest.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index d1bc86c3ef..89689306a8 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -9,3 +9,4 @@ pub mod runs; pub mod scheduler; pub mod snapshots; pub mod store; +pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index a3af3bf96e..598797bfc5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -9,30 +9,29 @@ use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_cron_time::next_cron_time; use crate::scheduler::main::parse_cron::parse_cron; use crate::scheduler::main::previous_cron_time::previous_cron_time; -use crate::scheduler::models::CatchUp; +use crate::scheduler::models::{CatchUp, ScheduleExpansion}; use crate::store::errors::ControlError; pub(crate) fn expand_schedule( trigger: &TriggerDefinition, from: DateTime, now: DateTime, -) -> Result<(Vec>, Option>), ControlError> { - let Schedule::Cron { - expression, - timezone, - } = &trigger.schedule - else { - let Schedule::Calendar { times, .. } = &trigger.schedule else { - unreachable!() - }; - return Ok(( - times.range((Excluded(from), Unbounded)).copied().collect(), - None, - )); +) -> Result { + let (expression, timezone) = match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => (expression, timezone), + Schedule::Calendar { times, .. } => { + return Ok(( + times.range((Excluded(from), Unbounded)).copied().collect(), + None, + )); + } }; let cron = parse_cron(expression, timezone)?; let latest = trigger.catch_up == CatchUp::Latest; - let mut due = Vec::new(); + let mut due: Vec> = Vec::new(); let mut cursor = from; for _ in 0..(MAX_TICKS_PER_BATCH - usize::from(latest)) { cursor = next_cron_time(&cron, cursor)?; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index a3a0c2ccad..b219340219 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -46,6 +46,9 @@ pub struct CronSchedule { pub(crate) timezone: chrono_tz::Tz, } +/// Expanded wall-clock times and the first omitted time, if the batch was bounded. +pub(crate) type ScheduleExpansion = (Vec>, Option>); + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum CatchUp { diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs new file mode 100644 index 0000000000..cbac454f9b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -0,0 +1,49 @@ +//! A worker's authenticated HTTP requests carry only its own run and claim. + +use uuid::Uuid; + +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::runs::models::{RunChange, RunChangeRequest, RunExecution}; +use crate::worker::_helpers::response::response; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +impl WorkerClient { + /// Fetch the launcher's immutable request, not the location's latest deployment. + pub async fn execution(&self) -> Result { + let execution: RunExecution = + response(self.client.get(format!("{}execution", self.run_url))).await?; + if execution.claim.request.run_id != self.run_id + || execution.claim.claim_id != self.claim_id + { + return Err(WorkerError::Configuration( + "execution response does not match the worker's run and claim", + )); + } + Ok(execution) + } + + /// Call with the same event ID and payload when retrying an uncertain response. + pub async fn report(&self, event_id: Uuid, change: RunChange) -> Result { + let request = RunChangeRequest { + event_id, + run_id: self.run_id, + claim_id: Some(self.claim_id), + change, + }; + let result: MutationResponse = response( + self.client + .post(format!("{}events", self.run_url)) + .json(&request), + ) + .await?; + Ok(result.applied) + } + + /// Run independently from warehouse execution; cancellation is a cooperative request. + pub async fn heartbeat(&self) -> Result { + let result: HeartbeatResponse = + response(self.client.post(format!("{}heartbeat", self.run_url))).await?; + Ok(result.cancel_requested) + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs new file mode 100644 index 0000000000..57d94ff592 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod client; +pub(crate) mod response; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs new file mode 100644 index 0000000000..e4da90cca5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs @@ -0,0 +1,15 @@ +//! Do not echo response bodies: server diagnostics can contain user configuration. + +use serde::de::DeserializeOwned; + +use crate::worker::errors::WorkerError; + +pub(crate) async fn response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await.map_err(reqwest::Error::without_url)?; + if !response.status().is_success() { + return Err(WorkerError::Rejected(response.status())); + } + Ok(response.json().await.map_err(reqwest::Error::without_url)?) +} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs new file mode 100644 index 0000000000..30a6a18f8e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -0,0 +1,15 @@ +//! Transport failures are distinct from rejected pinned execution inputs. + +#[derive(Debug, thiserror::Error)] +pub enum WorkerError { + #[error("invalid worker connection: {0}")] + Configuration(&'static str), + #[error("worker API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("worker API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), + #[error(transparent)] + Verification(#[from] crate::store::errors::ControlError), + #[error("run cancellation was requested before execution")] + Cancelled, +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs new file mode 100644 index 0000000000..f1c543d1dc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -0,0 +1,54 @@ +//! Bound worker requests have deadlines and must never follow credential-bearing redirects. + +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use uuid::Uuid; + +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +fn worker_url(server_url: &str, run_id: Uuid) -> Result { + let mut url = reqwest::Url::parse(server_url) + .map_err(|_| WorkerError::Configuration("server URL must be an absolute HTTP URL"))?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(WorkerError::Configuration( + "server URL must use HTTP or HTTPS without credentials, query or fragment", + )); + } + let prefix = url.path().trim_end_matches('/'); + url.set_path(&format!("{prefix}/api/runs/{run_id}/")); + Ok(url) +} + +/// Connect to an API origin or path prefix using the launcher's run-scoped bearer. +pub fn connect_worker( + server_url: &str, + bearer: &str, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let run_url = worker_url(server_url, run_id)?; + let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) + .map_err(|_| WorkerError::Configuration("run bearer is not a valid HTTP credential"))?; + authorization.set_sensitive(true); + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + let client = reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?; + Ok(WorkerClient { + client, + run_url, + run_id, + claim_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/mod.rs b/crates/sqlbuild-orchestration/src/worker/main/mod.rs new file mode 100644 index 0000000000..05f5586133 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connect_worker; +pub mod start_worker; diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs new file mode 100644 index 0000000000..c5710f3e31 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -0,0 +1,28 @@ +//! Verify the baseline publication before any run overrides or warehouse mutation. + +use uuid::Uuid; + +use crate::runs::models::{RunChange, RunExecution}; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::SnapshotInput; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +/// Verify the baseline compile before planning, cloning, applying run settings or executing. +pub async fn start_worker( + client: &WorkerClient, + baseline: &SnapshotInput, +) -> Result { + let expected = client.execution().await?; + verify_execution(baseline, &expected)?; + if client.heartbeat().await? { + return Err(WorkerError::Cancelled); + } + client + .report( + Uuid::new_v5(&expected.claim.claim_id, b"worker-started"), + RunChange::Started, + ) + .await?; + Ok(expected) +} diff --git a/crates/sqlbuild-orchestration/src/worker/mod.rs b/crates/sqlbuild-orchestration/src/worker/mod.rs new file mode 100644 index 0000000000..59f273cbec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs new file mode 100644 index 0000000000..813b4b61e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -0,0 +1,11 @@ +//! A worker owns only an API credential, never a control-store connection. + +use uuid::Uuid; + +#[derive(Clone, Debug)] +pub struct WorkerClient { + pub(crate) client: reqwest::Client, + pub(crate) run_url: reqwest::Url, + pub(crate) run_id: Uuid, + pub(crate) claim_id: Uuid, +} diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 9ee75a0eb6..14ce5705f3 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -10,3 +10,5 @@ mod store_test_types; mod test_contract; #[path = "api/test_types.rs"] mod test_types; +#[path = "api/test_worker.rs"] +mod test_worker; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index c3231b7060..5ebd106605 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -9,6 +9,11 @@ pub(crate) struct ApiContractTestCase { pub(crate) expected_status: RunStatus, } +pub(crate) struct WorkerRedirectTestCase { + pub(crate) description: &'static str, + pub(crate) expected_status: reqwest::StatusCode, +} + pub(crate) struct TestApi { pub(crate) url: String, pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs new file mode 100644 index 0000000000..dd61d4724c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -0,0 +1,356 @@ +use std::collections::BTreeMap; + +use axum::Router; +use axum::response::Redirect; +use axum::routing::get; +use chrono::Utc; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::run_helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{ApiContractTestCase, TestApi, WorkerRedirectTestCase}; + +#[test] +fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() + { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker HTTP lifecycle", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + ApiContractTestCase { + description: "PostgreSQL worker HTTP lifecycle", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let baseline = snapshot(location); + let publication = publish_snapshot(store, &baseline).await.expect("publish"); + queue( + store, + &request((location, publication.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let identity = (claim.request.run_id, claim.claim_id); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: identity.0, + claim_id: identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let worker = connect_worker(&api.url, &token.bearer, identity).expect("worker client"); + assert!( + !format!("{worker:?}").contains(&token.bearer), + "{}", + test_case.description + ); + let wrong_claim = connect_worker(&api.url, &token.bearer, (identity.0, Uuid::now_v7())) + .expect("different local claim"); + assert!( + matches!( + wrong_claim.execution().await, + Err(WorkerError::Configuration(_)) + ), + "{}", + test_case.description + ); + let mut drifted = baseline.clone(); + drifted + .nodes + .get_mut("inventory") + .expect("unselected dependency") + .raw_sql = Some("select 2 as id".into()); + assert!( + matches!( + start_worker(&worker, &drifted).await, + Err(WorkerError::Verification(_)) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("unstarted"); + assert_eq!( + runs[0].status, + RunStatus::Queued, + "{}", + test_case.description + ); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: identity.0, + claim_id: Some(identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("record process launch"); + let execution = start_worker(&worker, &baseline) + .await + .expect("verified start"); + assert_eq!(execution.claim, claim, "{}", test_case.description); + assert_eq!( + start_worker(&worker, &baseline).await.expect("retry start"), + execution, + "{}", + test_case.description + ); + assert!( + !worker.heartbeat().await.expect("heartbeat"), + "{}", + test_case.description + ); + let started = Uuid::now_v7(); + assert!( + worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("node start"), + "{}", + test_case.description + ); + assert!( + !worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("retry node start"), + "{}", + test_case.description + ); + worker + .report( + Uuid::now_v7(), + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(7), + error_code: None, + skipped_because_of: None, + }, + ) + .await + .expect("node outcome"); + worker + .report( + Uuid::now_v7(), + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "warehouse work completed".into(), + exit_code: Some(0), + }, + ) + .await + .expect("finish"); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("completed"); + assert_eq!( + runs[0].status, test_case.expected_status, + "{}", + test_case.description + ); + assert!( + matches!( + worker.heartbeat().await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + queue( + store, + &request((location, publication.snapshot_id), 2, 0, &["orders"]), + ) + .await; + let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim another run") + .expect("queued work"); + let cancelled_identity = (cancelled_claim.request.run_id, cancelled_claim.claim_id); + let cancelled_token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: cancelled_identity.0, + claim_id: cancelled_identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("another worker credential"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("another launched process"); + change_run( + store, + RunAuthority::Operator("test-operator"), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + }, + }, + ) + .await + .expect("cancel before execution"); + let cancelled_worker = + connect_worker(&api.url, &cancelled_token.bearer, cancelled_identity) + .expect("cancelled worker connection"); + assert!( + matches!( + start_worker(&cancelled_worker, &baseline).await, + Err(WorkerError::Cancelled) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("cancelled run has not started"); + let cancelled_run = runs + .iter() + .find(|run| run.run_id == cancelled_identity.0) + .expect("cancelled projection"); + assert_eq!( + cancelled_run.status, + RunStatus::Launched, + "{}", + test_case.description + ); + assert!( + cancelled_run.started_at.is_none(), + "{}", + test_case.description + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_not_followed() { + let test_cases = [WorkerRedirectTestCase { + description: "a run bearer must not be forwarded by an API redirect", + expected_status: reqwest::StatusCode::TEMPORARY_REDIRECT, + }]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated redirect server"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let application = Router::new().route( + "/api/runs/{run_id}/execution", + get(|| async { Redirect::temporary("/must-not-receive-a-credential") }), + ); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, application) + .with_graceful_shutdown(async { + let _ = signal.await; + }) + .await + .map_err(ControlError::from) + }); + let api = TestApi { + url, + shutdown, + task, + }; + let client = connect_worker( + &api.url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("worker client"); + let Err(WorkerError::Rejected(status)) = client.execution().await else { + panic!("{}", test_case.description); + }; + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + stop_api(api).await; + }); + } +} From b1515048338913df135bf9d35f556bbb5583ae85 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:54:04 +0000 Subject: [PATCH 004/138] fix: scope orchestration test helpers to their consumers Separate lease expiry, replica connections, terminal run fixtures and fault cases so each integration binary imports only the support it uses. Preserve SQLite and PostgreSQL transaction coverage. --- .../tests/credentials.rs | 6 +++- .../tests/credentials/test_contract.rs | 6 ++-- crates/sqlbuild-orchestration/tests/recall.rs | 2 ++ .../tests/recall/helpers.rs | 3 +- crates/sqlbuild-orchestration/tests/runs.rs | 8 ++++- .../tests/runs/helpers.rs | 30 ++----------------- .../tests/runs/terminal_helpers.rs | 30 +++++++++++++++++++ .../tests/runs/test_claims.rs | 6 ++-- .../tests/runs/test_faults.rs | 3 +- .../sqlbuild-orchestration/tests/scheduler.rs | 4 +++ .../tests/scheduler/helpers.rs | 27 ++--------------- .../sqlbuild-orchestration/tests/snapshots.rs | 2 ++ .../tests/snapshots/helpers.rs | 3 +- .../tests/store/fault_types.rs | 10 +++++++ .../tests/store/helpers.rs | 16 ---------- .../tests/store/lease_helpers.rs | 24 +++++++++++++++ .../tests/store/replica_helpers.rs | 15 ++++++++++ crates/sqlbuild-orchestration/tests/ticks.rs | 6 ++++ .../tests/ticks/fault_helpers.rs | 27 ++--------------- .../tests/ticks/helpers.rs | 6 ++-- .../tests/ticks/test_loop.rs | 5 ++-- .../tests/ticks/test_types.rs | 10 +------ 22 files changed, 135 insertions(+), 114 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/fault_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/lease_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/replica_helpers.rs diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs index a4b17699db..c40545cc9e 100644 --- a/crates/sqlbuild-orchestration/tests/credentials.rs +++ b/crates/sqlbuild-orchestration/tests/credentials.rs @@ -1,12 +1,16 @@ #[path = "ticks/fault_helpers.rs"] mod fault_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "runs/helpers.rs"] mod run_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; +#[path = "runs/terminal_helpers.rs"] +mod terminal_helpers; #[path = "credentials/test_contract.rs"] mod test_contract; -#[path = "ticks/test_types.rs"] +#[path = "store/fault_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 6768c12897..9b0ff859f2 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -15,9 +15,11 @@ use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; -use crate::run_helpers::{finish, queue, request, snapshot}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::replica_helpers::second_store; +use crate::run_helpers::{queue, request, snapshot}; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::terminal_helpers::finish; use crate::test_types::TickFaultTestCase; #[test] diff --git a/crates/sqlbuild-orchestration/tests/recall.rs b/crates/sqlbuild-orchestration/tests/recall.rs index 274b96b9ed..3261be8a4e 100644 --- a/crates/sqlbuild-orchestration/tests/recall.rs +++ b/crates/sqlbuild-orchestration/tests/recall.rs @@ -1,5 +1,7 @@ #[path = "recall/helpers.rs"] mod helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/recall/helpers.rs index 72e824e82a..ce7de75ac3 100644 --- a/crates/sqlbuild-orchestration/tests/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/recall/helpers.rs @@ -6,7 +6,8 @@ use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::event_object_row::event_object_row; use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index f2b42df0c2..627c22a308 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -2,10 +2,16 @@ mod fault_helpers; #[path = "runs/helpers.rs"] mod helpers; +#[path = "store/lease_helpers.rs"] +mod lease_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; +#[path = "runs/terminal_helpers.rs"] +mod terminal_helpers; #[path = "runs/test_claims.rs"] mod test_claims; #[path = "runs/test_coalesce.rs"] @@ -14,5 +20,5 @@ mod test_coalesce; mod test_faults; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; -#[path = "ticks/test_types.rs"] +#[path = "store/fault_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index e98f62b1df..586e14c21e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -3,12 +3,9 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::json; -use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunKind, RunOrigin, RunOutcome, RunRequest, -}; -use sqlbuild_orchestration::scheduler::models::{Overlap, SchedulerLease}; +use sqlbuild_orchestration::runs::models::{RunKind, RunOrigin, RunRequest}; +use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; @@ -93,26 +90,3 @@ pub(crate) async fn queue(store: &ControlStore, request: &RunRequest) { "a reused run ID cannot change its request" ); } - -pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { - let request = RunChangeRequest { - event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), - run_id: claim.request.run_id, - claim_id: Some(claim.claim_id), - change: RunChange::Finished { - outcome: RunOutcome::Failed, - reason: "synthetic worker exit".into(), - exit_code: Some(1), - }, - }; - assert!( - change_run(store, RunAuthority::Scheduler(lease), &request) - .await - .expect("terminate claimed work") - ); - assert!( - !change_run(store, RunAuthority::Scheduler(lease), &request) - .await - .expect("retry terminal fact") - ); -} diff --git a/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs b/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs new file mode 100644 index 0000000000..68e8742d32 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs @@ -0,0 +1,30 @@ +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, +}; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "synthetic worker exit".into(), + exit_code: Some(1), + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("terminate claimed work") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("retry terminal fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 4489f0322d..b97dc931e5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -11,9 +11,11 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::helpers::{finish, queue, request, snapshot}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::helpers::{queue, request, snapshot}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::terminal_helpers::finish; #[test] fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 7ab8721043..a0f076c366 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -14,8 +14,9 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; +use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; +use crate::lease_helpers::expire_lease; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::TickFaultTestCase; diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs index 159c963ed1..be400b8236 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -1,5 +1,9 @@ #[path = "scheduler/helpers.rs"] mod helpers; +#[path = "store/lease_helpers.rs"] +mod lease_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index f1711e4bfc..8cac2b9924 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -4,34 +4,13 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::lease_helpers::expire_lease; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; -async fn expire_lease(store: &ControlStore) { - let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { - panic!("expected SQLite fixture"); - }; - rusqlite::Connection::open(path) - .expect("open fault injection connection") - .execute( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - [], - ) - .expect("expire SQLite scheduler lease"); - return; - }; - sqlx_core::query::query( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .execute(pool) - .await - .expect("expire PostgreSQL scheduler lease"); -} - pub(crate) async fn leadership_contract(backend: Backend) -> Option { let test_store = open_test_store(backend).await?; let store = &test_store.store; diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs index a554a78fc6..5dcb1cd22c 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -2,6 +2,8 @@ mod contract; #[path = "snapshots/helpers.rs"] mod helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index aaefbd5da1..b1533caba3 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -13,7 +13,8 @@ use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; use crate::test_types::SchemaRejectionCase; diff --git a/crates/sqlbuild-orchestration/tests/store/fault_types.rs b/crates/sqlbuild-orchestration/tests/store/fault_types.rs new file mode 100644 index 0000000000..eb9e82bd2f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/fault_types.rs @@ -0,0 +1,10 @@ +//! Transaction failure points shared by credential, tick and run contracts. + +pub(crate) struct TickFaultTestCase { + pub(crate) description: &'static str, + pub(crate) table: &'static str, + pub(crate) operation: &'static str, + pub(crate) condition: &'static str, + pub(crate) sqlite_body: &'static str, + pub(crate) postgres_body: &'static str, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 07aafd8e98..f93f44f0f5 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -78,19 +78,3 @@ pub(crate) async fn close_store(store: ControlStore) { }; pool.close().await; } - -pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { - let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { - panic!("expected SQLite fixture"); - }; - return open_store(path.to_str().expect("UTF-8 test path")) - .await - .expect("open independent SQLite handle"); - }; - ControlStore::Postgres( - sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) - .await - .expect("open independent PostgreSQL pool"), - ) -} diff --git a/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs b/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs new file mode 100644 index 0000000000..8e17cfce86 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs @@ -0,0 +1,24 @@ +//! Expire the persisted lease without introducing a fake control-store clock. + +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn expire_lease(store: &ControlStore) { + match store { + ControlStore::Sqlite(path) => { + rusqlite::Connection::open(path) + .expect("open lease expiry connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite scheduler lease"); + } + ControlStore::Postgres(pool) => { + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs b/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs new file mode 100644 index 0000000000..853ba75df8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs @@ -0,0 +1,15 @@ +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + match store { + ControlStore::Sqlite(path) => open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"), + ControlStore::Postgres(pool) => ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ), + } +} diff --git a/crates/sqlbuild-orchestration/tests/ticks.rs b/crates/sqlbuild-orchestration/tests/ticks.rs index 00bfc0cbb6..abf2590859 100644 --- a/crates/sqlbuild-orchestration/tests/ticks.rs +++ b/crates/sqlbuild-orchestration/tests/ticks.rs @@ -1,7 +1,13 @@ #[path = "ticks/fault_helpers.rs"] mod fault_helpers; +#[path = "store/fault_types.rs"] +mod fault_types; #[path = "ticks/helpers.rs"] mod helpers; +#[path = "store/lease_helpers.rs"] +mod lease_helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs index 180b1babce..9c973c0fb1 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs @@ -13,14 +13,14 @@ pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCas "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", point.operation, point.table, point.condition, point.sqlite_body )) - .expect("install SQLite transaction fault"); + .expect(point.description); return; }; sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) .execute(pool).await.expect("install PostgreSQL fault function"); sqlx_core::query::query(&format!("CREATE TRIGGER sqb_test_fault BEFORE {} ON {} FOR EACH ROW {} EXECUTE FUNCTION sqb_test_fault()", - point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + &[")", ""][usize::from(point.condition.is_empty())])) - .execute(pool).await.expect("install PostgreSQL transaction fault"); + point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + [")", ""][usize::from(point.condition.is_empty())])) + .execute(pool).await.expect(point.description); } pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase) { @@ -43,24 +43,3 @@ pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase .await .expect("remove PostgreSQL fault function"); } - -pub(crate) async fn expire_lease(store: &ControlStore) { - let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { - unreachable!() - }; - rusqlite::Connection::open(path) - .expect("fault connection") - .execute_batch( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .expect("expire SQLite lease"); - return; - }; - sqlx_core::query::query( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .execute(pool) - .await - .expect("expire PostgreSQL lease"); -} diff --git a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs index 183b105cde..0ce57192b9 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/helpers.rs @@ -15,8 +15,10 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{expire_lease, install_fault, remove_fault}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, second_store}; +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::lease_helpers::expire_lease; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; use crate::test_types::TickFaultTestCase; diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs index 34b691b8fa..d847e5546a 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs @@ -8,9 +8,10 @@ use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::fault_helpers::expire_lease; use crate::helpers::snapshot; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run, second_store}; +use crate::lease_helpers::expire_lease; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::TickTransactionTestCase; diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs index 05ee9cd1d5..9559e8a92f 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/test_types.rs @@ -1,3 +1,4 @@ +pub(crate) use crate::fault_types::TickFaultTestCase; use crate::store_test_types::Backend; pub(crate) struct TickTransactionTestCase { @@ -7,15 +8,6 @@ pub(crate) struct TickTransactionTestCase { pub(crate) expected_runs: usize, } -pub(crate) struct TickFaultTestCase { - pub(crate) description: &'static str, - pub(crate) table: &'static str, - pub(crate) operation: &'static str, - pub(crate) condition: &'static str, - pub(crate) sqlite_body: &'static str, - pub(crate) postgres_body: &'static str, -} - pub(crate) struct TriggerControlsTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, From 28f364d8753af8f649e742b4ffe33b0f834f77da Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:03:36 +0000 Subject: [PATCH 005/138] feat: derive orchestration snapshots from native compiler records --- Cargo.lock | 2 + .../sql/edge_cases/edge_next_quantity.sql | 8 + .../seeds/edge_cases/edge_quantity_steps.csv | 3 + .../seeds/edge_cases/edge_quantity_steps.yml | 7 + crates/sqlbuild-python/Cargo.toml | 2 + .../bindings/_helpers/compiled_project/mod.rs | 4 + .../_helpers/compiled_project/registration.rs | 3 +- .../_helpers/compiled_project/snapshot.rs | 104 +++++++ .../compiled_project/snapshot_boundary.rs | 183 +++++++++++++ .../compiled_project/snapshot_models.rs | 73 +++++ .../compiled_project/snapshot_resources.rs | 140 ++++++++++ .../_helpers/compiled_project/values.rs | 11 + .../compiled_views/compiled_model_view.rs | 30 +- .../_helpers/compiled_views/function_views.rs | 5 +- .../_helpers/compiled_views/seed_views.rs | 2 +- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_environment.rs | 50 ++++ src/sqlbuild/_native.pyi | 11 + .../compiler/compiled_project/_test_types.py | 7 + .../test_orchestration_snapshot.py | 256 ++++++++++++++++++ 20 files changed, 887 insertions(+), 15 deletions(-) create mode 100644 benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql create mode 100644 benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv create mode 100644 benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py diff --git a/Cargo.lock b/Cargo.lock index cef38f434c..92db47c581 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2624,6 +2624,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "sha2", "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", @@ -2635,6 +2636,7 @@ dependencies = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-recall", diff --git a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql new file mode 100644 index 0000000000..c4fa936dea --- /dev/null +++ b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql @@ -0,0 +1,8 @@ +FUNCTION ( + description 'Advance an order quantity in the inherited function namespace.', + tags [edge_case], + arguments (quantity INTEGER), + returns INTEGER, +); + +SELECT quantity + 1 diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv new file mode 100644 index 0000000000..9e70ca2ea3 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv @@ -0,0 +1,3 @@ +quantity +2 +7 diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml new file mode 100644 index 0000000000..de890aa0b8 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml @@ -0,0 +1,7 @@ +seeds: + - name: edge_quantity_steps + description: Quantities whose seed destination inherits the target namespace. + tags: [edge_case] + columns: + - name: quantity + type: INTEGER diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 661bd3d6a1..be0cc75d7c 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -24,6 +24,7 @@ rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -35,6 +36,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-recall.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index bbd4e1a61a..1fb46a0eb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -3,4 +3,8 @@ pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; pub(crate) mod schema_facts; +pub(crate) mod snapshot; +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_resources; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index fd619ffe5f..c342479217 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -6,5 +6,6 @@ use crate::bindings::_helpers::{compiled_views, input_loops}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; input_loops::registration::register(module)?; - compiled_views::registration::register(module) + compiled_views::registration::register(module)?; + crate::bindings::_helpers::compiled_project::snapshot::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs new file mode 100644 index 0000000000..1edaf8fcd4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -0,0 +1,104 @@ +//! Project metadata stays native until its immutable snapshot is sent over HTTP. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; + +use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::records::fields::field; + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationSnapshot { + pub(crate) input: SnapshotInput, +} + +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } +} + +/// Compile catalog nodes from their owners, never from Python model or graph projections. +#[pyfunction] +pub(crate) fn build_orchestration_snapshot( + py: Python<'_>, + project: PyRef<'_, NativeProjectView>, + metadata_json: &str, + project_dir: PathBuf, +) -> PyResult { + let mut input: SnapshotInput = serde_json::from_str(metadata_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !input.nodes.is_empty() { + return Err(PyValueError::new_err( + "snapshot nodes must come from the compiled project", + )); + } + if project + .effective_target_name() + .is_some_and(|target| target != input.target) + { + return Err(PyValueError::new_err( + "snapshot target differs from the compiled project", + )); + } + for diagnostic in project.diagnostics(py).bind(py).try_iter()? { + let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; + if severity == "error" { + return Err(PyValueError::new_err( + "cannot publish a project with compiler errors", + )); + } + } + input.sqb_version = env!("CARGO_PKG_VERSION").into(); + input.environment_hash = environment_hash(py, &project_dir)?; + let owner = project.owner().bind(py).borrow(); + input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + Ok(NativeOrchestrationSnapshot { input }) +} + +/// Duplicate resource identities must fail, rather than silently replacing an earlier node. +pub(crate) fn insert_node( + mut nodes: BTreeMap, + key: String, + node: SnapshotNode, +) -> PyResult> { + if nodes.insert(key, node).is_some() { + return Err(PyValueError::new_err("duplicate snapshot node identity")); + } + Ok(nodes) +} + +pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { + deps.iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect() +} + +pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { + serde_json::json!({ + "database": value.database, + "schema": value.schema, + "name": value.name, + "qualified_name": value.qualified_name, + "logical_database": value.logical_database, + "logical_schema": value.logical_schema, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs new file mode 100644 index 0000000000..8eaa09494a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs @@ -0,0 +1,183 @@ +//! Discovery files and authored test payloads remain the Python host's boundary. + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{destination, insert_node}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, SeedViews}; +use crate::bindings::_helpers::records::fields::{field, path_posix}; + +/// Read only declared metadata fields, never callable attributes or connection settings. +pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult { + let mut result = Map::new(); + for name in names { + result.insert((*name).into(), plain_value(&field(value, name)?)?); + } + Ok(Value::Object(result)) +} + +pub(crate) fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { + path_posix(&field(value, "relative_path")?) +} + +/// Source locations are local compiler diagnostics, not relocatable column metadata. +pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult> { + value + .try_iter()? + .map(|column| { + metadata( + &column?, + &["name", "type", "nullable", "description", "meta"], + ) + }) + .collect() +} + +fn file_digest(value: &Bound<'_, PyAny>) -> PyResult { + let path: PathBuf = field(value, "file_path")?.extract()?; + value.py().detach(|| { + let mut file = File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let length = file.read(&mut buffer)?; + if length == 0 { + return Ok(format!("{:x}", hash.finalize())); + } + hash.update(&buffer[..length]); + } + }) +} + +pub(crate) fn boundary_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + let seed_views = project.seeds(py); + for (seed, view) in owner.seeds().iter().zip(seed_views.bind(py).try_iter()?) { + let effective = view? + .cast::()? + .get() + .current_destination(py)?; + let parts = seed.parts(); + let seed_file = parts.seed_file.bind(py); + nodes = insert_node( + nodes, + format!("seed:{}", seed.name), + SnapshotNode { + kind: "seed".into(), + definition: json!({ + "name": seed.name, + "path": relative_path(seed_file)?, + "content_hash": file_digest(seed_file)?, + "destination": destination(&effective), + "schema": metadata(parts.schema_entry.bind(py), &[ + "tags", "description", "meta", "csv_settings", + ])?, + "columns": columns(&field(parts.schema_entry.bind(py), "columns")?)?, + }), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )?; + } + for (kind, objects, names) in [ + ( + "sql_test", + project.sql_tests(py), + ["mode", "payload", "parameter_schema", "parameter_values"].as_slice(), + ), + ( + "sql_scenario", + project.sql_scenarios(py), + ["authored_ctes", "expected_ctes", "assertion_ctes"].as_slice(), + ), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let key = field(&item, "key")?; + let name: String = field(&key, "name")?.extract()?; + let resource_type: String = field(&key, "resource_type")?.str()?.extract()?; + let definition = metadata(&item, names)?; + nodes = insert_node( + nodes, + format!("{resource_type}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "test": definition}), + raw_sql: None, + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + } + for (kind, objects) in [ + ("loader", project.loader_functions(py)), + ("hook", project.hook_functions(py)), + ("materialization", project.materialization_files(py)), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + let path: PathBuf = field(&item, "file_path")?.extract()?; + let source = py.detach(|| std::fs::read_to_string(path))?; + nodes = insert_node( + nodes, + format!("{kind}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(source), + }, + )?; + } + } + for item in project.sql_hook_files(py).bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + nodes = insert_node( + nodes, + format!("sql_hook:{name}"), + SnapshotNode { + kind: "sql_hook".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: Some(field(&item, "contents")?.extract()?), + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + let macros = project.loaded_macros(py); + for (key, item) in macros.bind(py).cast::()?.iter() { + let name: String = key.extract()?; + nodes = insert_node( + nodes, + format!("macro:{name}"), + SnapshotNode { + kind: "macro".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(field(&item, "raw_source")?.extract()?), + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs new file mode 100644 index 0000000000..13fa7aca8c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -0,0 +1,73 @@ +//! Model catalog data comes directly from the compiler's retained facts and records. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; +use pyo3::types::PyTupleMethods; +use serde_json::{Value, json}; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn model_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for model in project.models(py).bind(py).iter() { + let model = model.cast::()?.get(); + let (name, _, _) = model.file_texts(py)?; + let facts = model_facts(owner.facts(), &name) + .ok_or_else(|| PyValueError::new_err("snapshot model has no retained facts"))?; + if facts.config.encoding_error.is_some() { + return Err(PyValueError::new_err( + "model config cannot be represented in snapshot metadata", + )); + } + let mut definition = model.read(py, |input, _| { + let record = input + .compiled + .as_ref() + .ok_or_else(|| PyValueError::new_err("snapshot model has not been assembled"))?; + let columns: Vec = record + .analysis + .inferred + .as_deref() + .unwrap_or_default() + .iter() + .map(|(name, data_type, nullability)| { + json!({"name": name, "type": data_type, "nullability": nullability}) + }) + .collect(); + Ok(json!({ + "name": facts.name, + "path": facts.relative_path, + "config": facts.config.values, + "materialization": facts.config.materialized, + "deps": dependency_keys(&record.deps), + "columns": columns, + })) + })?; + definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; + nodes = insert_node( + nodes, + format!("model:{}", facts.name), + SnapshotNode { + kind: "model".into(), + definition, + raw_sql: Some(facts.authored_sql.to_string()), + compiled_sql: Some(facts.query_sql.to_string()), + python_source: None, + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs new file mode 100644 index 0000000000..213bcc9739 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs @@ -0,0 +1,140 @@ +//! Sources, functions and audits are read from their retained native records. + +use std::collections::BTreeMap; + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_project::snapshot_boundary::{ + boundary_nodes, columns, metadata, relative_path, +}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn resource_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for source in owner.sources() { + let attached = &source.attached; + nodes = insert_node( + nodes, + format!("source:{}", attached.name), + SnapshotNode { + kind: "source".into(), + definition: json!({ + "name": attached.name, + "path": relative_path(attached.host.source_file.bind(py))?, + "database": attached.database, + "schema": attached.schema, + "table": attached.table, + "expression": attached.expression, + "contract": attached.contract, + "description": attached.description, + "namespace": source.namespace, + "columns": columns(attached.host.columns.bind(py).as_any())?, + "meta": plain_value(attached.host.meta.bind(py))?, + "entry": metadata(attached.host.entry.bind(py), &[ + "loader", "write_strategy", "cursor_column", "unique_key", + "final", "load_batch_size", "freshness", + ])?, + }), + raw_sql: None, + compiled_sql: attached.expression.clone(), + python_source: None, + }, + )?; + } + let function_views = project.functions(py); + for (function, view) in owner + .functions() + .iter() + .zip(function_views.bind(py).try_iter()?) + { + let effective = view? + .cast::()? + .get() + .current_destinations(py)? + .0; + let attached = &function.attached; + let kind = attached.node_type(); + let python = attached.python.as_ref().map(|runtime| { + json!({ + "version": runtime.runtime_version, + "entry_point": runtime.entry_point, + "packages": runtime.packages, + }) + }); + nodes = insert_node( + nodes, + format!("{kind}:{}", attached.name), + SnapshotNode { + kind: kind.into(), + definition: json!({ + "name": attached.name, + "path": attached.relative_path, + "arguments": attached.arguments, + "returns": attached.returns, + "return_columns": attached.return_columns, + "destination": destination(&effective), + "python": python, + "tags": attached.tags, + "description": attached.description, + "deps": dependency_keys(function.deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: attached.python.is_none().then(|| attached.body.clone()), + python_source: attached.python.is_some().then(|| attached.body.clone()), + }, + )?; + } + for audit in owner.audits() { + let attached = &audit.attached; + let target: Option = attached.target.as_ref().map( + |target| json!({"kind": target.kind, "name": target.name, "column": target.column}), + ); + nodes = insert_node( + nodes, + format!("audit:{}", attached.compiled_name()), + SnapshotNode { + kind: "audit".into(), + definition: json!({ + "name": attached.compiled_name(), + "definition_name": attached.definition_name, + "measurement": attached.measurement, + "evidence_sql": attached.evidence_sql, + "target": target, + "severity": attached.severity, + "run_scope": attached.run_scope, + "always_run": attached.always_run, + "description": attached.description, + "policy": metadata(attached.host.audit_block.bind(py), &[ + "evaluation_mode", "measurement_contract", + ])?, + "thresholds": plain_value( + attached.host.thresholds.bind(py), + )?, + "minimum_samples": plain_value( + attached.host.minimum_samples.bind(py), + )?, + "evidence_limit": plain_value( + attached.host.evidence_limit.bind(py), + )?, + "deps": dependency_keys(audit.scope_deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: Some(attached.sql_body.clone()), + python_source: None, + }, + )?; + } + boundary_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs index 90a9f140d2..2a551bbd39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs @@ -9,6 +9,7 @@ use serde_json::{Map, Number, Value}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::models::NativeRecord; const DATACLASS_FIELDS: &str = "__dataclass_fields__"; const INSTANCE_DICT: &str = "__dict__"; @@ -49,6 +50,16 @@ pub(crate) fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(mapping) = value.downcast::() { return object_value(mapping); } + if let Ok(record) = value.cast::() { + let record = record.get(); + let mut entries = Vec::with_capacity(record.values.len()); + for (field, item) in record.spec.fields.iter().zip(&record.values) { + if !field.name.starts_with(PRIVATE_PREFIX) { + entries.push((field.name.to_owned(), plain_value(item.bind(value.py()))?)); + } + } + return Ok(sorted_object(entries)); + } object_like_value(value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 441e7d723d..a83881c415 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -260,6 +260,22 @@ impl NativeCompiledModel { self.read(py, |record, py| read(compiled_of(record)?, py)) } + /// Read the effective destination, including namespace overlays, without a Python projection. + pub(crate) fn read_destination( + &self, + py: Python<'_>, + read: impl FnOnce(&DestinationFact) -> PyResult, + ) -> PyResult { + self.compiled(py, |compiled, _| { + read( + self.overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination), + ) + }) + } + /// This view's typed analysis as the owner's model facts read it, and the model's name. pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { self.read(py, |record, _| { @@ -552,16 +568,11 @@ impl NativeCompiledModel { #[getter] pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { + self.read_destination(py, |destination| { self.projections .destination .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); relation_location(&classes.compiled, destination) }) .map(|destination| destination.clone_ref(py)) @@ -612,12 +623,7 @@ impl NativeCompiledModel { /// `(database, schema, name)` of the model's destination. #[getter] pub(crate) fn destination_row(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| { - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); + self.read_destination(py, |destination| { Ok(( destination.database.clone(), destination.schema.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs index 7be4f6bee8..af49d0e3f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs @@ -436,7 +436,10 @@ impl NativeCompiledFunction { } /// The destination and fingerprint destination, with target defaults once applied. - fn current_destinations(&self, py: Python<'_>) -> PyResult<(DestinationFact, DestinationFact)> { + pub(crate) fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs index b6f857e5f5..2d070f9535 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs @@ -237,7 +237,7 @@ impl NativeCompiledSeed { }) } - fn current_destination(&self, py: Python<'_>) -> PyResult { + pub(crate) fn current_destination(&self, py: Python<'_>) -> PyResult { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 8f9ded74e2..637d9ef489 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod connections; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; +pub(crate) mod orchestration_environment; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs new file mode 100644 index 0000000000..fac1f3356d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -0,0 +1,50 @@ +//! Read the location interpreter's installed dependencies, never its credentials or paths. + +use std::path::Path; + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use sha2::{Digest, Sha256}; + +/// A lockfile binds declared dependencies; without one, bind installed distribution versions. +pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { + let mut hash = Sha256::new(); + let version = py.version_info(); + hash.update(format!( + "sqlbuild:{}\npython:{}.{}.{}{}\n", + env!("CARGO_PKG_VERSION"), + version.major, + version.minor, + version.patch, + version.suffix.unwrap_or_default(), + )); + match py.detach(|| std::fs::read(project_dir.join("uv.lock"))) { + Ok(lockfile) => { + hash.update(b"uv.lock\n"); + hash.update(lockfile); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let mut packages = Vec::new(); + for distribution in py + .import("importlib.metadata")? + .call_method0("distributions")? + .try_iter()? + { + let distribution = distribution?; + let name: String = distribution + .getattr("metadata")? + .get_item("Name")? + .extract()?; + let version: String = distribution.getattr("version")?.extract()?; + packages.push((name.replace(['_', '.'], "-").to_ascii_lowercase(), version)); + } + packages.sort(); + hash.update(b"installed-packages\n"); + hash.update( + serde_json::to_vec(&packages) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?, + ); + } + Err(error) => return Err(error.into()), + } + Ok(format!("{:x}", hash.finalize())) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4a78cb2ea7..3452f57cab 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1738,6 +1738,17 @@ class NativeCompiledAudit: @property def meta(self) -> dict[str, object] | None: ... +class NativeOrchestrationSnapshot: + """A compiled data-only publication retained natively until transport.""" + + def to_json(self) -> str: ... + +def build_orchestration_snapshot( + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, +) -> NativeOrchestrationSnapshot: ... + class NativeProjectView: @property def run_id(self) -> str: ... diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index bfb3ca0eab..2d7ea4bbb5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -190,3 +190,10 @@ class QuotedHookBoundaryTestCase: description: str argument: str expected_sql: str + + +@dataclass(frozen=True) +class SnapshotCatalogTestCase: + description: str + files: dict[str, str] + expected_kinds: frozenset[str] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py new file mode 100644 index 0000000000..629fc0bdc1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -0,0 +1,256 @@ +"""Real compilation produces immutable, relocatable control-plane catalog data.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from uuid import UUID + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + SnapshotCatalogTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + FUNCTION_PROJECT_FILES, + PARITY_FILES, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + +METADATA: dict[str, Any] = { + "location_id": str(UUID(int=1)), + "target": "development", + "generation": 1, + "git_revision": None, + "git_dirty": False, + "git_author": None, + "image_id": None, + "environment_hash": "test-environment", + "sqb_version": "not-the-installed-version", + "nodes": {}, + "documents": {}, +} + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotCatalogTestCase( + "model and seed catalog", + TYPE_PROOF_FILES + | { + "seeds/payment_methods.csv": PARITY_FILES["seeds/payment_methods.csv"], + "seeds/payment_methods.yml": PARITY_FILES["seeds/payment_methods.yml"], + "models/payments.sql": ( + "MODEL (description 'Payment methods.');\n" + 'SELECT method FROM __seed("payment_methods")\n' + ), + }, + frozenset({"model", "seed"}), + ), + SnapshotCatalogTestCase( + "attached audits", + TYPE_PROOF_FILES + | { + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (audits [not_null])));\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "audit"}), + ), + SnapshotCatalogTestCase( + "SQL and Python functions", + { + path: value + for path, value in FUNCTION_PROJECT_FILES.items() + if path != "enums/status.sql" + } + | {"functions/sql/_enums/status.sql": FUNCTION_PROJECT_FILES["enums/status.sql"]}, + frozenset({"model", "udf", "table_fn"}), + ), + SnapshotCatalogTestCase( + "Python hooks", + TYPE_PROOF_FILES + | { + "hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n" + "@hook\ndef notify(ctx, channel):\n" + ' """Notify support."""\n return None\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [python('notify', channel: 'support')]);\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "hook"}), + ), + SnapshotCatalogTestCase( + "SQL hooks", + TYPE_PROOF_FILES + | { + "hooks/sql/record_load.sql": "HOOK (description 'Record load.');\nSELECT 3 AS load_count\n", + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [sql('record_load')]);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "sql_hook"}), + ), + SnapshotCatalogTestCase( + "source catalog", + TYPE_PROOF_FILES + | { + "sources/orders.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: raw\n table: orders\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + frozenset({"model", "source"}), + ), + SnapshotCatalogTestCase( + "DuckShop inherited seed and function namespaces", + TYPE_PROOF_FILES + | { + path: (Path("benchmarks/duckshop/project") / path).read_text() + for path in ( + "seeds/edge_cases/edge_quantity_steps.csv", + "seeds/edge_cases/edge_quantity_steps.yml", + "functions/sql/edge_cases/edge_next_quantity.sql", + ) + }, + frozenset({"model", "seed", "udf"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_and_relocatable( + test_case: SnapshotCatalogTestCase, + tmp_path: Path, +) -> None: + publications: list[dict[str, Any]] = [] + metadata: str = json.dumps(METADATA) + for directory in (tmp_path / "original", tmp_path / "relocated"): + write_files(project_dir=directory, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=directory), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not [item for item in project.diagnostics if item.severity == "error"] + snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, metadata, directory + ) + publication: dict[str, Any] = json.loads(snapshot.to_json()) + publications.append(publication) + assert publication["sqb_version"] != "not-the-installed-version" + assert publication["environment_hash"] != "test-environment" + assert test_case.expected_kinds <= {node["kind"] for node in publication["nodes"].values()} + assert str(directory) not in snapshot.to_json() + for model in project.models: + node: dict[str, Any] = publication["nodes"][f"model:{model.name}"] + assert node["compiled_sql"] == model.query_sql + assert ( + node["definition"]["destination"]["qualified_name"] + == model.destination.qualified_name + ) + assert set(node["definition"]["deps"]) == { + f"{dependency.resource_type}:{dependency.name}" for dependency in model.deps + } + for function in project.functions: + assert ( + publication["nodes"][f"{function.key.resource_type}:{function.name}"]["definition"][ + "destination" + ]["qualified_name"] + == function.destination.qualified_name + ) + for seed in project.seeds: + assert ( + publication["nodes"][f"seed:{seed.name}"]["definition"]["destination"][ + "qualified_name" + ] + == seed.destination.qualified_name + ) + assert publications[0] == publications[1] + + +def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snapshot_is_emitted( + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with pytest.raises(ValueError, match="nodes must come from the compiled project"): + build_orchestration_snapshot( + project, + json.dumps( + METADATA + | { + "nodes": { + "model:forged": { + "kind": "model", + "definition": {}, + "raw_sql": None, + "compiled_sql": "select 2", + "python_source": None, + } + } + } + ), + tmp_path, + ) + (tmp_path / "models/orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") + broken: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert any(item.severity == "error" for item in broken.diagnostics) + with pytest.raises(ValueError, match="cannot publish a project with compiler errors"): + build_orchestration_snapshot(broken, json.dumps(METADATA), tmp_path) + + +def test_given_a_dependency_lock_when_it_changes_then_environment_identity_changes_not_node_content( + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + installed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision one\n") + locked: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision two\n") + changed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + assert ( + len( + {installed["environment_hash"], locked["environment_hash"], changed["environment_hash"]} + ) + == 3 + ) + assert installed["nodes"] == locked["nodes"] == changed["nodes"] + assert changed == json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) From 7c507a287f2138829a844388cd35a5778d5152f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:19:40 +0000 Subject: [PATCH 006/138] refactor: clarify scheduler storage ownership and regression cases --- .../src/identifier_text/main/mod.rs | 1 + .../identifier_text/main/quote_identifier.rs | 6 + .../src/runs/_helpers/claims.rs | 2 +- .../src/runs/_helpers/nodes.rs | 7 +- .../src/runs/constants.rs | 4 + .../src/runs/main/change_run.rs | 3 +- crates/sqlbuild-orchestration/src/runs/mod.rs | 1 + .../src/scheduler/_helpers/expansion.rs | 3 +- .../src/scheduler/main/run_scheduler.rs | 3 +- .../src/scheduler/main/scheduler_cycle.rs | 8 +- .../src/scheduler/mod.rs | 1 + .../src/scheduler/models.rs | 3 - .../src/scheduler/types.rs | 6 + .../src/store/main/read_recall_view.rs | 4 +- crates/sqlbuild-orchestration/tests/runs.rs | 4 +- .../tests/runs/test_claims.rs | 101 ++++++++++--- .../tests/runs/test_coalesce.rs | 21 ++- .../tests/runs/test_faults.rs | 42 ++++-- .../tests/runs/test_lifecycle.rs | 134 +++++++++++------- .../tests/runs/test_types.rs | 44 ++++++ .../tests/ticks/test_loop.rs | 13 +- .../_helpers/compiled_project/snapshot.rs | 3 +- .../native_host/orchestration_environment.rs | 2 +- .../src/clickhouse/_helpers/literals.rs | 3 +- .../src/clickhouse/constants.rs | 2 - .../src/clickhouse/main/encode_rows.rs | 3 +- .../src/records/_helpers/rows.rs | 11 -- crates/sqlbuild-recall/src/records/models.rs | 14 ++ .../src/schema/_helpers/postgres_views.rs | 14 +- .../src/schema/_helpers/relational_tables.rs | 23 +-- .../src/schema/_helpers/relational_views.rs | 48 +++---- .../src/schema/_helpers/rendering.rs | 5 - .../sqlbuild-recall/src/schema/constants.rs | 5 +- .../src/schema/main/add_column_statement.rs | 5 +- .../schema/main/control_view_statements.rs | 3 +- .../main/postgres_partition_statement.rs | 2 +- .../src/schema/main/sqlite_retention_sql.rs | 3 +- .../src/schema/main/upgrade_statements.rs | 3 +- .../src/schema/main/view_statements.rs | 4 +- crates/sqlbuild-recall/src/schema/models.rs | 16 +++ .../src/store/_helpers/relational.rs | 3 +- .../src/store/_helpers/rows.rs | 4 +- .../src/store/main/conflict_evidence.rs | 8 +- .../src/store/main/relational_insert.rs | 12 +- .../src/store/main/relational_lookup.rs | 18 ++- fensu.toml | 12 ++ .../compiler/compiled_project/_test_types.py | 17 +++ .../test_orchestration_snapshot.py | 84 +++++++---- 48 files changed, 491 insertions(+), 247 deletions(-) create mode 100644 crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/types.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_types.rs diff --git a/crates/sqlbuild-core/src/identifier_text/main/mod.rs b/crates/sqlbuild-core/src/identifier_text/main/mod.rs index a29dc793b1..b9d09a0ef5 100644 --- a/crates/sqlbuild-core/src/identifier_text/main/mod.rs +++ b/crates/sqlbuild-core/src/identifier_text/main/mod.rs @@ -1 +1,2 @@ pub mod is_lowercase_identifier; +pub mod quote_identifier; diff --git a/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs new file mode 100644 index 0000000000..efd258cbc0 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs @@ -0,0 +1,6 @@ +//! Standard SQL double-quoted identifiers, including embedded quote escaping. + +#[must_use] +pub fn quote_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs index 2b65b19559..8e69205b5f 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs @@ -9,6 +9,7 @@ use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Occupied slots above a lowered limit still count against the run's concurrency group. pub(crate) async fn reserve_claims( mut transaction: ControlTransaction, request: &RunRequest, @@ -62,7 +63,6 @@ pub(crate) async fn reserve_claims( .iter() .filter_map(|row| row.get("slot").and_then(serde_json::Value::as_i64)) .collect(); - // Lowering a limit also counts occupied slots above the new maximum. if occupied.len() >= limit as usize { return Ok((transaction, false)); } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs index cb8d430a89..8ae44b1a94 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use serde_json::json; +use crate::runs::constants::{NODE_PENDING, NODE_RUNNING}; use crate::runs::models::{RunChange, RunChangeRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -15,7 +16,7 @@ pub(crate) async fn project_node( counts: &BTreeMap, ) -> Result { let (node, status, rows, error, skipped) = match &request.change { - RunChange::NodeStarted { node_key } => (node_key, "running", None, None, None), + RunChange::NodeStarted { node_key } => (node_key, NODE_RUNNING, None, None, None), RunChange::NodeFinished { node_key, outcome, @@ -47,7 +48,7 @@ pub(crate) async fn project_node( .and_then(|row| row.get("status")) .and_then(serde_json::Value::as_str) .ok_or_else(|| ControlError::Invalid("node is outside the frozen run selection".into()))?; - if previous != "pending" && (previous != "running" || status == "running") { + if previous != NODE_PENDING && (previous != NODE_RUNNING || status == NODE_RUNNING) { return Err(ControlError::Invalid( "node change is inconsistent with its current outcome".into(), )); @@ -67,7 +68,7 @@ pub(crate) async fn project_node( let (transaction, _) = query(transaction, "UPDATE sqb_run_nodes SET status=$1, started_at=CASE WHEN $1='running' THEN $2 ELSE started_at END, \ ended_at=$3, rows_affected=$4, error_code=$5, skipped_because_of=$6 WHERE run_id=$7 AND node_key=$8 AND attempt=1", - vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != "running").then_some(now)), + vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != NODE_RUNNING).then_some(now)), rows.map_or(Parameter::Null(sqlbuild_recall::schema::models::ColumnType::Integer), Parameter::Integer), Parameter::OptionalText(error), Parameter::OptionalText(skipped), Parameter::Id(request.run_id), Parameter::Text(node.clone())]).await?; let (transaction, _) = query( diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs new file mode 100644 index 0000000000..d92a2e052a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -0,0 +1,4 @@ +//! Nonterminal node states; terminal states come from NodeOutcome. + +pub(crate) const NODE_PENDING: &str = "pending"; +pub(crate) const NODE_RUNNING: &str = "running"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index f3884cb69f..458ae8ae4a 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -14,7 +14,7 @@ use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -/// Authority comes from authenticated API state, not client-supplied actor or scope fields. +/// Use authenticated authority and record unclaimed cancellation as terminal in the same transaction. pub async fn change_run( store: &ControlStore, authority: RunAuthority<'_>, @@ -67,7 +67,6 @@ pub async fn change_run( ) .await?; let mut transaction = project_change(transaction, request, &state).await?; - // No process can exist before a durable claim. Cancel queued work without leaving it inert forever. if matches!(request.change, RunChange::CancelRequested { .. }) && state.worker_instance.is_none() { diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs index 5359000d16..f147865646 100644 --- a/crates/sqlbuild-orchestration/src/runs/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index 598797bfc5..8ba7d7e422 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -9,7 +9,8 @@ use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_cron_time::next_cron_time; use crate::scheduler::main::parse_cron::parse_cron; use crate::scheduler::main::previous_cron_time::previous_cron_time; -use crate::scheduler::models::{CatchUp, ScheduleExpansion}; +use crate::scheduler::models::CatchUp; +use crate::scheduler::types::ScheduleExpansion; use crate::store::errors::ControlError; pub(crate) fn expand_schedule( diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 4bf8917b6f..29c8180b65 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -15,7 +15,8 @@ pub async fn run_scheduler( mut changes: watch::Receiver<()>, ) -> Result<(), ControlError> { while !*shutdown.borrow() { - let cycle = scheduler_cycle(store, &mut state).await?; + let (next_state, cycle) = scheduler_cycle(store, state).await?; + state = next_state; tokio::select! { _ = tokio::time::sleep(cycle.wait_for) => {}, result = changes.changed() => { if result.is_err() { break; } }, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs index 9e6edf9a32..011d28e303 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs @@ -18,8 +18,8 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn scheduler_cycle( store: &ControlStore, - state: &mut SchedulerState, -) -> Result { + mut state: SchedulerState, +) -> Result<(SchedulerState, SchedulerCycle), ControlError> { let renewal = match state.lease.as_ref() { Some(lease) => renew_lease(store, lease).await.map(Some), None => acquire_lease(store, state.holder).await, @@ -36,7 +36,7 @@ pub async fn scheduler_cycle( wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), }; let Some(mut lease) = state.lease.clone() else { - return Ok(cycle); + return Ok((state, cycle)); }; let transaction = begin(store, TransactionMode::Read).await?; let now = transaction.now; @@ -71,5 +71,5 @@ pub async fn scheduler_cycle( .min(cycle.wait_for); } commit(transaction).await?; - Ok(cycle) + Ok((state, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs index d686433c8a..3978df0a4a 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index b219340219..a3a0c2ccad 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -46,9 +46,6 @@ pub struct CronSchedule { pub(crate) timezone: chrono_tz::Tz, } -/// Expanded wall-clock times and the first omitted time, if the batch was bounded. -pub(crate) type ScheduleExpansion = (Vec>, Option>); - #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum CatchUp { diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs new file mode 100644 index 0000000000..274b453740 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -0,0 +1,6 @@ +//! Compound scheduler return values. + +use chrono::{DateTime, Utc}; + +/// Expanded wall-clock times and the first omitted time, if the batch was bounded. +pub(crate) type ScheduleExpansion = (Vec>, Option>); diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs index 4c323fcd1b..a07da01efb 100644 --- a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs @@ -9,7 +9,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::store::types::Row; -/// The API authorizes the invocation and evidence capability before entering this storage reader. +/// Read authorized evidence, decoding only display text while preserving byte-exact captured content. pub async fn read_recall_view( store: &ControlStore, (view, invocation_id): (&str, &str), @@ -28,8 +28,6 @@ pub async fn read_recall_view( ], ) .await?; - // These PostgreSQL view columns retain bytes: conversion to database text would reject NUL. - // Decode only the display field, leaving captured content and transcripts byte-exact. if matches!(view, "output_tail" | "artifacts") { for row in &mut rows { if let Some(text) = row.get_mut("text") diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 627c22a308..10a2813d22 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -1,5 +1,7 @@ #[path = "ticks/fault_helpers.rs"] mod fault_helpers; +#[path = "store/fault_types.rs"] +mod fault_types; #[path = "runs/helpers.rs"] mod helpers; #[path = "store/lease_helpers.rs"] @@ -20,5 +22,5 @@ mod test_coalesce; mod test_faults; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; -#[path = "store/fault_types.rs"] +#[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index b97dc931e5..509232d7f2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -16,13 +16,30 @@ use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::terminal_helpers::finish; +use crate::test_types::{BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase}; #[test] fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + PriorityClaimsTestCase { + description: "SQLite claims prioritize independent write sets and release atomically", + backend: Backend::Sqlite, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + PriorityClaimsTestCase { + description: "PostgreSQL claims prioritize independent write sets and release atomically", + backend: Backend::Postgres, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -61,10 +78,13 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol .collect(); assert_eq!( claimed, - [Uuid::from_u128(2), Uuid::from_u128(3)] + test_case + .expected_initial_claims .into_iter() + .map(Uuid::from_u128) .collect(), - "higher priority wins the shared write set" + "{}", + test_case.description ); assert!( claim_next_run(store, &lease, &limits) @@ -102,14 +122,18 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol .iter() .filter(|event| event["event_type"] == "run_claimed") .count(), - 4 + test_case.expected_claim_events, + "{}", + test_case.description ); assert_eq!( events .iter() .filter(|event| event["event_type"] == "run_terminated") .count(), - 2 + test_case.expected_terminal_events, + "{}", + test_case.description ); let runs = list_runs(store, (location, "development"), 100) .await @@ -129,9 +153,21 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol #[test] fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_still_count_and_terminal_cleanup_unblocks_work() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite occupied high slots still count after lowering a group limit", + backend: Backend::Sqlite, + expected_unblocked_run: 3, + }, + GroupLimitTestCase { + description: "PostgreSQL occupied high slots still count after lowering a group limit", + backend: Backend::Postgres, + expected_unblocked_run: 3, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -194,7 +230,12 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil .await .expect("released group") .expect("third queued run"); - assert_eq!(third.request.run_id, Uuid::from_u128(3)); + assert_eq!( + third.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); close_test_store(fixture).await; }); } @@ -203,9 +244,21 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil #[test] fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_independent_work_is_not_starved() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + BacklogClaimTestCase { + description: "SQLite scans past more than a full page of blocked work", + backend: Backend::Sqlite, + expected_independent_run: 105, + }, + BacklogClaimTestCase { + description: "PostgreSQL scans past more than a full page of blocked work", + backend: Backend::Postgres, + expected_independent_run: 105, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -227,15 +280,15 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in .await .expect("claim blocker") .expect("available"); - for id in 2..=104 { - queue(store, &request(scope, id, 0, &["orders"])).await; - if id % 20 == 0 { - lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( - store, &lease, - ) - .await - .expect("keep leader live while preparing backlog"); + for batch in (2..=104).collect::>().chunks(20) { + for id in batch { + queue(store, &request(scope, *id, 0, &["orders"])).await; } + lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( + store, &lease, + ) + .await + .expect("keep leader live while preparing backlog"); } queue(store, &request(scope, 105, -1, &["customers"])).await; let started = std::time::Instant::now(); @@ -243,10 +296,16 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in .await .expect("scan past a full blocked page") .expect("independent work"); - assert_eq!(independent.request.run_id, Uuid::from_u128(105)); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(test_case.expected_independent_run), + "{}", + test_case.description + ); assert!( started.elapsed() < std::time::Duration::from_secs(5), - "claim scan exceeded five seconds on {backend:?}" + "claim scan exceeded five seconds: {}", + test_case.description ); close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index c05d357583..8c0ea844dd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -15,13 +15,26 @@ use uuid::Uuid; use crate::helpers::snapshot; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; +use crate::test_types::CoalescedRunTestCase; #[test] fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + CoalescedRunTestCase { + description: "SQLite coalesces only work that has not been claimed", + backend: Backend::Sqlite, + expected_runs: 2, + }, + CoalescedRunTestCase { + description: "PostgreSQL coalesces only work that has not been claimed", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -95,7 +108,9 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks .await .expect("one executing and one waiting") .len(), - 2 + test_case.expected_runs, + "{}", + test_case.description ); close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index a0f076c366..3332a4bd5c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -19,14 +19,26 @@ use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::TickFaultTestCase; +use crate::test_types::{ClaimFailureTestCase, TickFaultTestCase}; #[test] fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + ClaimFailureTestCase { + description: "SQLite rolls back every claim-publication failure window", + backend: Backend::Sqlite, + expected_run: 1, + }, + ClaimFailureTestCase { + description: "PostgreSQL rolls back every claim-publication failure window", + backend: Backend::Postgres, + expected_run: 1, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -59,21 +71,17 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene let projected = list_runs(store, (location, "development"), 100) .await .expect("queued projection"); - for table in [ - "sqb_write_claims", - "sqb_concurrency_slots", - "sqb_run_liveness", - "recall_events", - "sqb_runs", + for (table, operation) in [ + ("sqb_write_claims", "INSERT"), + ("sqb_concurrency_slots", "INSERT"), + ("sqb_run_liveness", "INSERT"), + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), ] { let point = TickFaultTestCase { description: "claim publication failure", table, - operation: if table == "sqb_runs" { - "UPDATE" - } else { - "INSERT" - }, + operation, condition: "", sqlite_body: "SELECT RAISE(ABORT,'injected claim failure')", postgres_body: "RAISE EXCEPTION 'injected claim failure'", @@ -121,6 +129,12 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene .expect("retry complete claim") .expect("no leaked claims or slots"); assert_eq!(claim.request, input); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_run), + "{}", + test_case.description + ); assert!( claim_next_run(store, &lease, &limits) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 7ed2c6874a..d61b50e82a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -19,14 +19,26 @@ use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::TickFaultTestCase; +use crate::test_types::{RunLifecycleTestCase, TickFaultTestCase}; #[test] fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves facts, outcomes and claims through worker failures", + backend: Backend::Sqlite, + expected_status: RunStatus::Partial, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves facts, outcomes and claims through worker failures", + backend: Backend::Postgres, + expected_status: RunStatus::Partial, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -137,42 +149,40 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ }, ..started.clone() }; - if node == "customers" { - let before_nodes = list_run_nodes(store, started.run_id) - .await - .expect("node before fault"); - let before_events = read_recall_events(store, 0, 100) - .await - .expect("events before node fault"); - for table in ["sqb_run_nodes", "sqb_runs"] { - let point = TickFaultTestCase { - description: "node projection and counter failure", - table, - operation: "UPDATE", - condition: "", - sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", - postgres_body: "RAISE EXCEPTION 'injected node failure'", - }; - install_fault(store, &point).await; - assert!( - change_run(store, worker, &completed).await.is_err(), - "{} at {table}", - point.description - ); - remove_fault(store, &point).await; - assert_eq!( - list_run_nodes(store, started.run_id) - .await - .expect("node rollback"), - before_nodes - ); - assert_eq!( - read_recall_events(store, 0, 100) - .await - .expect("node fact rollback"), - before_events - ); - } + let before_nodes = list_run_nodes(store, started.run_id) + .await + .expect("node before fault"); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("events before node fault"); + for table in ["sqb_run_nodes", "sqb_runs"] { + let point = TickFaultTestCase { + description: "node projection and counter failure", + table, + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &completed).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_run_nodes(store, started.run_id) + .await + .expect("node rollback"), + before_nodes + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("node fact rollback"), + before_events + ); } assert!( change_run(store, worker, &completed) @@ -201,22 +211,16 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ let before = read_recall_events(store, 0, 100) .await .expect("before terminal"); - for table in [ - "recall_events", - "sqb_runs", - "sqb_write_claims", - "sqb_run_liveness", + for (table, operation) in [ + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), + ("sqb_write_claims", "DELETE"), + ("sqb_run_liveness", "DELETE"), ] { let point = TickFaultTestCase { description: "terminal transaction failure", table, - operation: if table == "recall_events" { - "INSERT" - } else if table.starts_with("sqb_run") && table != "sqb_run_liveness" { - "UPDATE" - } else { - "DELETE" - }, + operation, condition: "", sqlite_body: "SELECT RAISE(ABORT,'injected terminal failure')", postgres_body: "RAISE EXCEPTION 'injected terminal failure'", @@ -317,7 +321,11 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .into_iter() .find(|run| run.run_id == started.run_id) .expect("original run"); - assert_eq!(projected.status, RunStatus::Partial); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); assert_eq!( projected.node_counts_json, BTreeMap::from([("success".into(), 1), ("failed".into(), 1)]) @@ -342,9 +350,21 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ #[test] fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_launch_and_preserves_the_actor() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite cancellation before launch preserves the operator and terminal fact", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL cancellation before launch preserves the operator and terminal fact", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -397,7 +417,11 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l .await .expect("cancelled projection") .remove(0); - assert_eq!(projected.status, RunStatus::Cancelled); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); assert_eq!( projected.node_counts_json, BTreeMap::from([("cancelled".into(), 1)]) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs new file mode 100644 index 0000000000..61c94312d4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -0,0 +1,44 @@ +//! Expected queue outcomes for each real control-store backend. + +use sqlbuild_orchestration::runs::models::RunStatus; + +pub(crate) use crate::fault_types::TickFaultTestCase; +use crate::store_test_types::Backend; + +pub(crate) struct PriorityClaimsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_initial_claims: [u128; 2], + pub(crate) expected_claim_events: usize, + pub(crate) expected_terminal_events: usize, +} + +pub(crate) struct GroupLimitTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_unblocked_run: u128, +} + +pub(crate) struct BacklogClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_independent_run: u128, +} + +pub(crate) struct CoalescedRunTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} + +pub(crate) struct ClaimFailureTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_run: u128, +} + +pub(crate) struct RunLifecycleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs index d847e5546a..2864da8921 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs @@ -46,30 +46,31 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain .await .expect("publish due schedule"); let second = second_store(store).await; - let mut first_state = SchedulerState::new(Uuid::now_v7()); - let mut second_state = SchedulerState::new(Uuid::now_v7()); - let first = scheduler_cycle(store, &mut first_state) + let first_state = SchedulerState::new(Uuid::now_v7()); + let second_state = SchedulerState::new(Uuid::now_v7()); + let (first_state, first) = scheduler_cycle(store, first_state) .await .expect("initial cycle"); assert!(first.leader, "{}", test_case.description); assert_eq!(first.evaluated, 1); assert_eq!(first.ticks, test_case.expected_ticks); assert_eq!(first.wait_for, Duration::from_secs(5)); - let follower = scheduler_cycle(&second, &mut second_state) + let (second_state, follower) = scheduler_cycle(&second, second_state) .await .expect("follower cycle"); assert!(!follower.leader); assert_eq!(follower.ticks, 0); expire_lease(store).await; - let successor = scheduler_cycle(&second, &mut second_state) + let (second_state, successor) = scheduler_cycle(&second, second_state) .await .expect("takeover cycle"); assert!(successor.leader); assert_eq!(successor.ticks, 0, "takeover never repeats durable work"); assert!( - !scheduler_cycle(store, &mut first_state) + !scheduler_cycle(store, first_state) .await .expect("fenced cycle") + .1 .leader ); let runs = sqlbuild_orchestration::runs::main::list_runs::list_runs( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 1edaf8fcd4..b0b0d878fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -7,6 +7,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; +use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; @@ -55,7 +56,7 @@ pub(crate) fn build_orchestration_snapshot( } for diagnostic in project.diagnostics(py).bind(py).try_iter()? { let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; - if severity == "error" { + if severity == DiagnosticSeverity::Error.as_str() { return Err(PyValueError::new_err( "cannot publish a project with compiler errors", )); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs index fac1f3356d..5b49bcaa07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -23,7 +23,7 @@ pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { - let mut packages = Vec::new(); + let mut packages: Vec<(String, String)> = Vec::new(); for distribution in py .import("importlib.metadata")? .call_method0("distributions")? diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 784b5e1cd3..a6934e5168 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -1,7 +1,6 @@ //! ClickHouse literals and key expressions for the store's own queries. use crate::clickhouse::constants::TIMESTAMP_PATTERN; -use crate::records::_helpers::rows; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::parse_timestamp::parse_timestamp; use crate::records::main::timestamp_micros::timestamp_micros; @@ -63,6 +62,6 @@ pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) } - _ => rows::key_text(value), + _ => value.map_or_else(String::new, RecallValue::key_text), } } diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index d11b50dc61..2c8f2719df 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -61,8 +61,6 @@ pub(crate) const MAX_DATETIME64_PRECISION: u64 = 9; pub(crate) const MICROS_DIGITS: u64 = 6; /// One decimal digit of timestamp scale. pub(crate) const DECIMAL_SCALE: i64 = 10; -/// The column the store sets when a row is written. -pub(crate) const INGESTED_AT_COLUMN: &str = "ingested_at"; pub(crate) const VARINT_PAYLOAD_BITS: u32 = 7; pub(crate) const VARINT_CONTINUE: u8 = 0x80; pub(crate) const VARINT_MASK: u64 = 0x7f; diff --git a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs index a91f7a0e35..7bd512fa2a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs +++ b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs @@ -1,9 +1,10 @@ //! `encode_rows`: one table's rows as a `RowBinary` insert block. use crate::clickhouse::_helpers::row_binary::{encoded, scaled}; -use crate::clickhouse::constants::{INGESTED_AT_COLUMN, MICROS_DIGITS}; +use crate::clickhouse::constants::MICROS_DIGITS; use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::INGESTED_AT_COLUMN; /// The column names and `RowBinary` block of `rows`, all of one table, written with `ingested_at`. Values for columns the table lacks are left out; columns the rows lack take their defaults. pub fn encode_rows( diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs index f5b60dc1e5..5ad2768b37 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -23,17 +23,6 @@ pub(crate) fn boolean(value: bool) -> RecallValue { RecallValue::Integer(i64::from(value)) } -/// The human-readable part of a conflict key, without dialect-specific timestamp normalization. -pub(crate) fn key_text(value: Option<&RecallValue>) -> String { - match value { - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), - } -} - /// `values` plus the envelope columns and `content_hash`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. pub(crate) fn finish_row( table: Table, diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index e9da820c87..e99a1cf280 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -12,6 +12,20 @@ pub enum RecallValue { Blob(Vec), } +impl RecallValue { + /// Human-readable conflict-key text, before dialect-specific timestamp normalization. + #[must_use] + pub fn key_text(&self) -> String { + match self { + Self::Text(text) => text.clone(), + Self::Integer(number) => number.to_string(), + Self::Real(number) => number.to_string(), + Self::Blob(_) => "".to_owned(), + Self::Null => String::new(), + } + } +} + /// One row bound for a Recall table: named values, without `ingested_at`, which the store sets. #[derive(Clone, Debug, PartialEq)] pub struct RecallRow { diff --git a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs index 98515a0e19..ece90e302a 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs @@ -29,12 +29,12 @@ fn json_fields(mut body: String) -> String { &format!("(payload ->> '{field}'){cast}"), ); } - for field in ["name", "job_name", "run_id", "retry_number"] { - let cast = if field == "retry_number" { - "::BIGINT" - } else { - "" - }; + for (field, cast) in [ + ("name", ""), + ("job_name", ""), + ("run_id", ""), + ("retry_number", "::BIGINT"), + ] { body = body.replace( &format!("json_extract(context.external_context, '$.integration.{field}')"), &format!("(context.external_context #>> '{{integration,{field}}}'){cast}"), @@ -50,6 +50,7 @@ fn json_fields(mut body: String) -> String { ) } +/// Keep display bytes until readers decode them: PostgreSQL text rejects NUL and malformed UTF-8. fn postgres_body(body: String) -> String { let mut body = json_fields(body) .replace("strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", "CURRENT_TIMESTAMP") @@ -59,7 +60,6 @@ fn postgres_body(body: String) -> String { .replace("MAX(COALESCE(received.queue_drops", "GREATEST(COALESCE(received.queue_drops") .replace("COALESCE(context.recall_filter_active, 0)", "COALESCE(CAST(context.recall_filter_active AS INTEGER), 0)") .replace("COALESCE(delivery.flush_complete, 0)", "COALESCE(CAST(delivery.flush_complete AS INTEGER), 0)") - // PostgreSQL text cannot hold NUL or malformed UTF-8. Keep display bytes until the reader decodes them. .replace("CAST(data AS TEXT)", "data") .replace("CAST(artifact.content AS TEXT)", "artifact.content") .replace(")\nGROUP BY", ") AS source_rows\nGROUP BY") diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs index 3930030bd3..5430d20356 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -1,19 +1,10 @@ //! One logical schema and index list for SQLite and PostgreSQL. -use crate::schema::_helpers::rendering::{all_columns, sqlite_identifier, sqlite_type}; -use crate::schema::models::{ColumnType, TableSpec}; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; -pub(crate) fn postgres_type(column: ColumnType) -> &'static str { - match column { - ColumnType::Text => "TEXT", - ColumnType::Timestamp => "TIMESTAMPTZ", - ColumnType::Json => "JSONB", - ColumnType::Integer => "BIGINT", - ColumnType::Boolean => "BOOLEAN", - ColumnType::Real => "DOUBLE PRECISION", - ColumnType::Blob => "BYTEA", - } -} +use crate::schema::_helpers::rendering::{all_columns, sqlite_type}; +use crate::schema::constants::{ARTIFACTS_TABLE, EVENTS_TABLE}; +use crate::schema::models::{ColumnType, TableSpec}; pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { let mut definitions: Vec = all_columns(spec) @@ -21,7 +12,7 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) .map(|column| { let not_null = if column.nullable { "" } else { " NOT NULL" }; let kind = if postgres { - postgres_type(column.column_type) + column.column_type.postgres_type() } else if control && column.column_type == ColumnType::Json { "JSON" } else { @@ -30,7 +21,7 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) }) .collect(); - if control && spec.name == "recall_events" { + if control && spec.name == EVENTS_TABLE.name { definitions.push(if postgres { " storage_id BIGINT GENERATED ALWAYS AS IDENTITY".into() } else { @@ -38,7 +29,7 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) }); } let mut key: Vec<&str> = spec.key.to_vec(); - let partition = if postgres && spec.name != "recall_artifacts" { + let partition = if postgres && spec.name != ARTIFACTS_TABLE.name { if !key.contains(&spec.expiry_column) { key.push(spec.expiry_column); } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 015c491b5e..c2bffe4987 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -212,30 +212,6 @@ SELECT END AS complete FROM counted"; -fn output_transcripts(postgres: bool) -> String { - let columns: Vec = [ - ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), - ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), - ("data", "combined"), - ] - .into_iter() - .map(|(expression, alias)| { - let value = if postgres { - format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") - } else { - format!( - "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" - ) - }; - format!(" {value} AS {alias}") - }) - .collect(); - format!( - "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", - columns.join(",\n") - ) -} - const EVENTS: &str = "SELECT * FROM recall_events"; const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; @@ -261,6 +237,30 @@ LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +fn output_transcripts(postgres: bool) -> String { + let columns: Vec = [ + ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), + ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), + ("data", "combined"), + ] + .into_iter() + .map(|(expression, alias)| { + let value = if postgres { + format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") + } else { + format!( + "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" + ) + }; + format!(" {value} AS {alias}") + }) + .collect(); + format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", + columns.join(",\n") + ) +} + fn invocations() -> String { format!( "WITH ids AS ( diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs index 658aef267d..d7f3eb635b 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -36,11 +36,6 @@ pub(crate) fn clickhouse_type(column: &ColumnSpec) -> String { } } -/// `"name"` for SQLite. -pub(crate) fn sqlite_identifier(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - /// `` `name` `` for ClickHouse. pub(crate) fn clickhouse_identifier(name: &str) -> String { format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 48e5a69a36..05c9c15d98 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -17,13 +17,16 @@ pub const DDL_SCRIPT_INVOCATION: &str = "ddl-script"; pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; +/// Assigned by storage rather than accepted from a producer's row values. +pub const INGESTED_AT_COLUMN: &str = "ingested_at"; + /// Columns every Recall row carries, after the table's own columns. pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("invocation_id", ColumnType::Text), ColumnSpec::required("invocation_started_at", ColumnType::Timestamp), ColumnSpec::required("schema_version", ColumnType::Integer), ColumnSpec::required("producer_version", ColumnType::Text), - ColumnSpec::required("ingested_at", ColumnType::Timestamp), + ColumnSpec::required(INGESTED_AT_COLUMN, ColumnType::Timestamp), ColumnSpec::required("content_hash", ColumnType::Text), ]; /// The artifact columns its content digest covers: what it is, not who saw it when. diff --git a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs index cea0815488..dc69488373 100644 --- a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs +++ b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs @@ -1,8 +1,9 @@ //! `add_column_statement`: the additive upgrade of one table by one column. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::rendering::{ - clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_identifier, - sqlite_type, + clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_type, }; use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, TableSpec}; diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs index 7b27ec1a2f..826dd58587 100644 --- a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -1,7 +1,8 @@ //! Control stores expose Recall's supported views on both engines. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::postgres_views::postgres_view_bodies; -use crate::schema::_helpers::rendering::sqlite_identifier; use crate::schema::main::view_statements::view_statements; use crate::schema::models::{DdlOptions, Dialect}; diff --git a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs index 971a3ae792..4d24b5fbdb 100644 --- a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs +++ b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs @@ -1,8 +1,8 @@ //! Monthly partition bounds are calculated from a typed date, never interpolated user SQL. use chrono::{Datelike, Months, NaiveDate}; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; -use crate::schema::_helpers::rendering::sqlite_identifier; use crate::schema::main::table_spec::table_spec; use crate::schema::models::Table; diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs index 1a44c7b2e4..44b5d5ee02 100644 --- a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -1,6 +1,7 @@ //! `sqlite_retention_sql`: the deletes SQLite maintenance runs, all rows of an invocation together. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::constants::{ INVOCATION_TABLES, META_MAINTENANCE_COMPONENT, RETENTION_CANDIDATE_TABLES, }; diff --git a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs index 0b8c7430e2..913bf845df 100644 --- a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs @@ -1,6 +1,7 @@ //! `upgrade_statements`: what an administrator runs to bring an observed store up to this build's schema. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::version_row::version_row_statement; use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION}; use crate::schema::main::add_column_statement::add_column_statement; diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index 8527aae75c..0351938b81 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -1,8 +1,10 @@ //! `view_statements`: the statements that create the supported views. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; use crate::schema::_helpers::relational_views::relational_view_bodies; -use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; +use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster}; use crate::schema::models::{DdlOptions, Dialect}; /// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 022e4f0b84..5da839a462 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -22,6 +22,22 @@ pub enum ColumnType { Boolean, } +impl ColumnType { + /// The PostgreSQL type used by both DDL and typed query placeholders. + #[must_use] + pub const fn postgres_type(self) -> &'static str { + match self { + Self::Text => "TEXT", + Self::Timestamp => "TIMESTAMPTZ", + Self::Json => "JSONB", + Self::Integer => "BIGINT", + Self::Boolean => "BOOLEAN", + Self::Real => "DOUBLE PRECISION", + Self::Blob => "BYTEA", + } + } +} + /// One column: its name, type and whether it may be null. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ColumnSpec { diff --git a/crates/sqlbuild-recall/src/store/_helpers/relational.rs b/crates/sqlbuild-recall/src/store/_helpers/relational.rs index becce06752..7122ec88b9 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/relational.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/relational.rs @@ -1,11 +1,10 @@ //! Typed placeholders shared by native SQLite and control-plane transactions. -use crate::schema::_helpers::relational_tables::postgres_type; use crate::schema::models::ColumnType; pub(crate) fn placeholder(index: usize, kind: ColumnType, postgres: bool) -> String { if postgres { - format!("CAST(${index} AS {})", postgres_type(kind)) + format!("CAST(${index} AS {})", kind.postgres_type()) } else { format!("${index}") } diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index a253f2ce78..aa5b4d0b42 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -5,7 +5,6 @@ use std::collections::{HashMap, HashSet}; use rusqlite::{Connection, OptionalExtension}; -use crate::records::_helpers::rows::key_text; use crate::records::main::conflict_row::conflict_row; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::main::table_spec::table_spec; @@ -19,7 +18,8 @@ use crate::store::main::relational_lookup::relational_lookup; use crate::store::models::{ConflictEvidence, InsertOutcome}; fn text_of(row: &RecallRow, column: &str) -> String { - key_text(row.value(column)) + row.value(column) + .map_or_else(String::new, RecallValue::key_text) } /// Insert one row's values for the columns its table has; `Some(evidence)` when its key holds different content. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs index a343086c2e..a4c04a92e6 100644 --- a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -1,7 +1,6 @@ //! Duplicate rows preserve the first content and expose the rejected digest. -use crate::records::_helpers::rows::key_text; -use crate::records::models::RecallRow; +use crate::records::models::{RecallRow, RecallValue}; use crate::schema::main::table_spec::table_spec; use crate::store::models::ConflictEvidence; @@ -14,7 +13,10 @@ pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option>() .join(":"), existing_sha256: existing.to_owned(), diff --git a/crates/sqlbuild-recall/src/store/main/relational_insert.rs b/crates/sqlbuild-recall/src/store/main/relational_insert.rs index ef63916e65..70aa341299 100644 --- a/crates/sqlbuild-recall/src/store/main/relational_insert.rs +++ b/crates/sqlbuild-recall/src/store/main/relational_insert.rs @@ -2,8 +2,10 @@ use std::collections::HashSet; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::records::models::{RecallRow, RecallValue}; -use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::constants::INGESTED_AT_COLUMN; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; use crate::schema::models::ColumnType; @@ -18,18 +20,18 @@ pub fn relational_insert<'a>( postgres: bool, ) -> RecallStatement<'a> { let metadata = table_columns(row.table); - let mut names = Vec::new(); - let mut values = Vec::new(); + let mut names: Vec = Vec::new(); + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); for (name, value) in &row.values { if let Some(column) = metadata.iter().find(|column| column.name == *name) && columns.contains(*name) - && *name != "ingested_at" + && *name != INGESTED_AT_COLUMN { names.push(sqlite_identifier(name)); values.push((value, column.column_type)); } } - names.push("ingested_at".into()); + names.push(INGESTED_AT_COLUMN.into()); values.push((ingested_at, ColumnType::Timestamp)); let placeholders: Vec = values .iter() diff --git a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs index d1291fdb4c..8d6658a6d2 100644 --- a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs +++ b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs @@ -1,10 +1,11 @@ //! Duplicate detection uses the same stable keys as physical unique constraints. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::records::models::{RecallRow, RecallValue}; -use crate::schema::_helpers::rendering::sqlite_identifier; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; -use crate::schema::models::Table; +use crate::schema::models::{ColumnType, Table}; use crate::store::_helpers::relational::placeholder; use crate::store::models::RecallStatement; @@ -17,14 +18,11 @@ pub fn relational_lookup(row: &RecallRow, postgres: bool, control: bool) -> Reca keys.push(spec.expiry_column); } let metadata = table_columns(row.table); - let mut values = Vec::new(); - let mut conditions = Vec::new(); - for key in keys { - let kind = metadata - .iter() - .find(|column| column.name == key) - .expect("Recall key columns belong to their schema") - .column_type; + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); + let mut conditions: Vec = Vec::new(); + for column in metadata.iter().filter(|column| keys.contains(&column.name)) { + let key = column.name; + let kind = column.column_type; values.push((row.value(key).unwrap_or(&RecallValue::Null), kind)); conditions.push(format!( "{} = {}", diff --git a/fensu.toml b/fensu.toml index 2f97466395..2338c3e56a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1564,6 +1564,18 @@ rule = "FPRSS201" path = "crates/sqlbuild-python/src/bindings/models.rs" reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/credentials/models.rs" +symbols = ["IssuedToken"] +reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." + +[[targets.rust.rule_exceptions]] +rule = "XSB072" +path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs" +symbols = ["NativeOrchestrationSnapshot"] +reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only SnapshotInput; binding functions remain checked." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 2d7ea4bbb5..6f5b1c5053 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -197,3 +197,20 @@ class SnapshotCatalogTestCase: description: str files: dict[str, str] expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotRejectionTestCase: + description: str + files: dict[str, str] + nodes: dict[str, object] + expected_compiler_errors: bool + expected_message: str + + +@dataclass(frozen=True) +class SnapshotEnvironmentTestCase: + description: str + first_lockfile: str + second_lockfile: str + expected_environment_count: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 629fc0bdc1..e53269d8a3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -19,6 +19,8 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, + SnapshotEnvironmentTestCase, + SnapshotRejectionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, @@ -148,7 +150,7 @@ def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_an adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) - assert not [item for item in project.diagnostics if item.severity == "error"] + assert not any(item.severity == "error" for item in project.diagnostics) snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( project, metadata, directory ) @@ -185,46 +187,70 @@ def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_an assert publications[0] == publications[1] +@pytest.mark.parametrize( + "test_case", + [ + SnapshotRejectionTestCase( + description="a valid compile cannot publish caller-forged nodes", + files=TYPE_PROOF_FILES, + nodes={ + "model:forged": { + "kind": "model", + "definition": {}, + "raw_sql": None, + "compiled_sql": "select 2", + "python_source": None, + } + }, + expected_compiler_errors=False, + expected_message="nodes must come from the compiled project", + ), + SnapshotRejectionTestCase( + description="compiler errors cannot replace the last good publication", + files=TYPE_PROOF_FILES | {"models/orders.sql": "MODEL ();\nSELECT 1 AS order_id\n"}, + nodes={}, + expected_compiler_errors=True, + expected_message="cannot publish a project with compiler errors", + ), + ], + ids=lambda case: case.description, +) def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snapshot_is_emitted( + test_case: SnapshotRejectionTestCase, tmp_path: Path, ) -> None: - write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + write_files(project_dir=tmp_path, files=test_case.files) project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) - with pytest.raises(ValueError, match="nodes must come from the compiled project"): + assert ( + any(item.severity == "error" for item in project.diagnostics) + == test_case.expected_compiler_errors + ) + with pytest.raises(ValueError, match=test_case.expected_message): build_orchestration_snapshot( project, - json.dumps( - METADATA - | { - "nodes": { - "model:forged": { - "kind": "model", - "definition": {}, - "raw_sql": None, - "compiled_sql": "select 2", - "python_source": None, - } - } - } - ), + json.dumps(METADATA | {"nodes": test_case.nodes}), tmp_path, ) - (tmp_path / "models/orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") - broken: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) - assert any(item.severity == "error" for item in broken.diagnostics) - with pytest.raises(ValueError, match="cannot publish a project with compiler errors"): - build_orchestration_snapshot(broken, json.dumps(METADATA), tmp_path) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotEnvironmentTestCase( + description="installed dependencies and two lock revisions identify different environments", + first_lockfile="version = 1\n# dependency revision one\n", + second_lockfile="version = 1\n# dependency revision two\n", + expected_environment_count=3, + ), + ], + ids=lambda case: case.description, +) def test_given_a_dependency_lock_when_it_changes_then_environment_identity_changes_not_node_content( + test_case: SnapshotEnvironmentTestCase, tmp_path: Path, ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) @@ -236,11 +262,11 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang installed: dict[str, Any] = json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) - (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision one\n") + (tmp_path / "uv.lock").write_text(test_case.first_lockfile) locked: dict[str, Any] = json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) - (tmp_path / "uv.lock").write_text("version = 1\n# dependency revision two\n") + (tmp_path / "uv.lock").write_text(test_case.second_lockfile) changed: dict[str, Any] = json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) @@ -248,7 +274,7 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang len( {installed["environment_hash"], locked["environment_hash"], changed["environment_hash"]} ) - == 3 + == test_case.expected_environment_count ) assert installed["nodes"] == locked["nodes"] == changed["nodes"] assert changed == json.loads( From 88cb5a921f2e54836eb0083c245e64685d2f4a82 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:31:49 +0000 Subject: [PATCH 007/138] refactor: separate scheduler policies from transactional coordinators --- .../src/native_commands/_helpers/decision.rs | 42 ++++++++++- .../src/native_commands/_helpers/execution.rs | 41 +++++++++++ .../src/native_commands/main/_run_control.rs | 72 ++----------------- .../src/declarations/_helpers/mod.rs | 1 + .../declarations/_helpers/trigger_state.rs | 27 +++++++ .../src/declarations/main/change_trigger.rs | 24 ++----- .../declarations/main/publish_definitions.rs | 29 +++----- .../src/declarations/mod.rs | 1 + .../src/runs/_helpers/changes.rs | 19 +++++ .../src/runs/main/change_run.rs | 13 +--- .../src/scheduler/_helpers/ticks.rs | 32 ++++++++- .../src/scheduler/main/plan_ticks.rs | 32 +-------- .../src/snapshots/_helpers/publication.rs | 39 +++++++++- .../src/snapshots/main/publish_snapshot.rs | 33 ++------- .../src/store/_helpers/migration.rs | 28 ++++++++ .../src/store/main/migrate_store.rs | 24 +------ 16 files changed, 258 insertions(+), 199 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 9355478a2d..21c6342413 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -1,7 +1,10 @@ -//! Which parsed invocations run without Python, decided from the parsed values alone. +//! Resolve native invocations and control-service settings from parsed arguments. +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; +use sqlbuild_orchestration::store::errors::ControlError; + use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; use crate::argv::models::{ArgvOutcome, Namespace, Value}; use crate::dispatch::main::invocation_policy::invocation_policy; @@ -36,6 +39,43 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } +pub(crate) fn control_address( + namespace: &Namespace, + standalone: bool, +) -> Result, ControlError> { + if standalone { + return Ok(None); + } + let host: IpAddr = text(namespace, "control_host") + .unwrap_or_else(|| "127.0.0.1".into()) + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let port = match namespace.get("control_port") { + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, + _ => 8000, + }; + Ok(Some(SocketAddr::new(host, port))) +} + +pub(crate) fn control_database(namespace: &Namespace) -> Result { + let variable = text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); + match std::env::var(&variable) { + Ok(value) => Ok(value), + Err(std::env::VarError::NotPresent) => Ok(PathBuf::from( + text(namespace, "project_dir").unwrap_or_else(|| ".".into()), + ) + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned()), + Err(std::env::VarError::NotUnicode(_)) => Err(ControlError::Invalid( + "control database variable must be UTF-8".into(), + )), + } +} + /// The strings of a list attribute, or `None` when it holds anything else. fn texts(namespace: &Namespace, name: &str) -> Option> { match namespace.get(name) { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 90297029d9..ac2f27678e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -4,6 +4,10 @@ use std::io; use std::path::{Path, PathBuf}; use std::time::SystemTime; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::watch; +use tokio::task::JoinSet; + use crate::argv::models::Stream; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; @@ -25,6 +29,43 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +async fn shutdown_signal() -> Result<(), ControlError> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; + Ok(()) +} + +/// Stop sibling roles on a signal or role failure, then drain their shutdown results. +pub(crate) async fn supervise_control_roles( + mut tasks: JoinSet>, + stop: watch::Sender, +) -> Result<(), ControlError> { + let mut result = tokio::select! { + result = tasks.join_next() => match result { + Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), + None => Ok(()), + }, + result = shutdown_signal() => result, + }; + stop.send_replace(true); + while let Some(joined) = tasks.join_next().await { + let joined = joined + .map_err(ControlError::from) + .and_then(std::convert::identity); + result = result.and(joined); + } + result +} + /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. fn open_capture( invocation: &NativeInvocation, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 111fc3a1a1..7de55c4455 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,8 +1,5 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. -use std::net::{IpAddr, SocketAddr}; -use std::path::PathBuf; - use sqlbuild_orchestration::api::main::serve_api::serve_api; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; @@ -12,8 +9,9 @@ use sqlbuild_orchestration::store::main::open_store::open_store; use tokio::sync::watch; use uuid::Uuid; -use crate::argv::models::{Namespace, Stream, Value}; -use crate::native_commands::_helpers::decision::{flag, text}; +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{control_address, control_database, flag}; +use crate::native_commands::_helpers::execution::supervise_control_roles; use crate::native_commands::_helpers::terminal::write_output; async fn stopped(mut shutdown: watch::Receiver) { @@ -24,56 +22,12 @@ async fn stopped(mut shutdown: watch::Receiver) { } } -async fn shutdown_signal() -> Result<(), ControlError> { - #[cfg(unix)] - { - let mut terminate = - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; - tokio::select! { - result = tokio::signal::ctrl_c() => result?, - _ = terminate.recv() => {}, - } - } - #[cfg(not(unix))] - tokio::signal::ctrl_c().await?; - Ok(()) -} - pub(crate) async fn run_control( namespace: &Namespace, standalone: bool, ) -> Result<(), ControlError> { - let address = if standalone { - None - } else { - let host: IpAddr = text(namespace, "control_host") - .unwrap_or_else(|| "127.0.0.1".into()) - .parse() - .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; - let port = match namespace.get("control_port") { - Some(Value::Int(value)) => value - .parse::() - .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, - _ => 8000, - }; - Some(SocketAddr::new(host, port)) - }; - let variable = text(namespace, "control_database_env") - .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); - let url = match std::env::var(&variable) { - Ok(value) => value, - Err(std::env::VarError::NotPresent) => { - PathBuf::from(text(namespace, "project_dir").unwrap_or_else(|| ".".into())) - .join(".sqlbuild/control.sqlite3") - .to_string_lossy() - .into_owned() - } - Err(std::env::VarError::NotUnicode(_)) => { - return Err(ControlError::Invalid( - "control database variable must be UTF-8".into(), - )); - } - }; + let address = control_address(namespace, standalone)?; + let url = control_database(namespace)?; drop(write_output(&[( Stream::Stderr, "Opening control store...\n", @@ -117,19 +71,5 @@ pub(crate) async fn run_control( "Scheduler loop started; leadership is lease-guarded\n", )])); } - let mut result = tokio::select! { - result = tasks.join_next() => match result { - Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), - None => Ok(()), - }, - result = shutdown_signal() => result, - }; - stop.send_replace(true); - while let Some(joined) = tasks.join_next().await { - let joined = joined - .map_err(ControlError::from) - .and_then(std::convert::identity); - result = result.and(joined); - } - result + supervise_control_roles(tasks, stop).await } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..40c4be79ad --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs new file mode 100644 index 0000000000..fccc6753f0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -0,0 +1,27 @@ +//! Recompute the effective schedule after code publication or an operator override. + +use chrono::{DateTime, TimeDelta, Utc}; + +use crate::declarations::models::{TriggerDefinition, TriggerStatus}; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::store::errors::ControlError; + +pub(crate) fn trigger_timing( + definition: &TriggerDefinition, + started_at: DateTime, + last_tick_for: Option>, + status_override: Option, +) -> Result<(TriggerStatus, Option>), ControlError> { + let status = status_override.unwrap_or(definition.default_status); + let after = last_tick_for.unwrap_or( + started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&definition.schedule, after)? + } else { + None + }; + Ok((status, next)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index e7d2165cf9..4ccd6765e0 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -1,11 +1,9 @@ //! Apply an operator fact and its projection in one transaction, with retry-safe request identity. -use chrono::TimeDelta; - +use crate::declarations::_helpers::trigger_state::trigger_timing; use crate::declarations::models::{ EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, }; -use crate::scheduler::main::next_schedule_time::next_schedule_time; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -60,20 +58,12 @@ pub async fn change_trigger( TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), TriggerChange::ResetCursor => state.last_tick_for = None, } - let status = state - .status_override - .unwrap_or(state.definition_json.default_status); - let after = state.last_tick_for.unwrap_or( - state - .started_at - .checked_sub_signed(TimeDelta::microseconds(1)) - .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, - ); - let next = if status == TriggerStatus::Running { - next_schedule_time(&state.definition_json.schedule, after)? - } else { - None - }; + let (status, next) = trigger_timing( + &state.definition_json, + state.started_at, + state.last_tick_for, + state.status_override, + )?; let now = transaction.now; let (transaction, _) = query(transaction, "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index c65b13468f..6110788d5e 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -1,10 +1,9 @@ //! Publish definition projections inside the current-snapshot transaction. -use chrono::TimeDelta; use uuid::Uuid; -use crate::declarations::models::{Definitions, PreviousTriggerState, TriggerStatus}; -use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::models::{Definitions, PreviousTriggerState}; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -67,24 +66,12 @@ pub(crate) async fn publish_definitions( trigger.starts_at.unwrap_or(snapshot.published_at), |state| state.started_at, ); - let default = trigger.default_status; - let status = state - .as_ref() - .and_then(|state| state.status_override) - .unwrap_or(default); - let after = state - .as_ref() - .and_then(|state| state.last_tick_for) - .unwrap_or( - started_at - .checked_sub_signed(TimeDelta::microseconds(1)) - .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, - ); - let next = if status == TriggerStatus::Running { - next_schedule_time(&trigger.schedule, after)? - } else { - None - }; + let (status, next) = trigger_timing( + &trigger, + started_at, + state.as_ref().and_then(|state| state.last_tick_for), + state.as_ref().and_then(|state| state.status_override), + )?; (transaction, _) = query(transaction, "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs index 5355003ff2..5359000d16 100644 --- a/crates/sqlbuild-orchestration/src/declarations/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod _helpers; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs index ba4b35be6e..e9480d69da 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs @@ -4,6 +4,25 @@ use crate::runs::models::{ ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; use crate::store::errors::ControlError; +use crate::store::types::Row; + +pub(crate) fn validate_retry( + previous: &Row, + request: &RunChangeRequest, + body: &serde_json::Value, +) -> Result<(), ControlError> { + if previous.get("payload") != Some(body) + || previous + .get("invocation_id") + .and_then(serde_json::Value::as_str) + != Some(format!("scheduler/{}", request.run_id).as_str()) + { + return Err(ControlError::Invalid( + "run change event ID already has different content".into(), + )); + } + Ok(()) +} pub(crate) fn validate_authority( authority: RunAuthority<'_>, diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 458ae8ae4a..3bef1033c8 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -2,7 +2,7 @@ use serde_json::json; -use crate::runs::_helpers::changes::{validate_authority, validate_change}; +use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; use crate::runs::_helpers::events::append_run_event; use crate::runs::_helpers::projection::project_change; use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome}; @@ -33,16 +33,7 @@ pub async fn change_run( ) .await?; if let Some(previous) = previous.first() { - if previous.get("payload") != Some(&body) - || previous - .get("invocation_id") - .and_then(serde_json::Value::as_str) - != Some(format!("scheduler/{}", request.run_id).as_str()) - { - return Err(ControlError::Invalid( - "run change event ID already has different content".into(), - )); - } + validate_retry(previous, request, &body)?; commit(transaction).await?; return Ok(false); } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs index 4cb2683e54..a4de72e3d6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -3,7 +3,37 @@ use chrono::{DateTime, SecondsFormat, Utc}; use uuid::Uuid; -use crate::scheduler::models::{TickAction, TickDecision}; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickState}; + +pub(crate) fn catch_up_actions(state: &TickState, due: &[DateTime]) -> Vec { + let covering_run = (state.overlap == Overlap::Coalesce) + .then_some(state.waiting_run) + .flatten(); + match (covering_run, state.catch_up) { + (Some(run_id), _) => due + .iter() + .map(|time| action(state.trigger_id, *time, Some(run_id))) + .collect(), + (None, CatchUp::All { max_runs }) => due + .iter() + .take(max_runs.get()) + .map(|time| action(state.trigger_id, *time, None)) + .collect(), + (None, CatchUp::Latest) => { + let latest = due.last().copied(); + let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); + due.iter() + .map(|time| { + action( + state.trigger_id, + *time, + (Some(*time) != latest).then_some(run_id).flatten(), + ) + }) + .collect() + } + } +} pub(crate) fn tick_id(trigger_id: Uuid, scheduled_for: DateTime) -> Uuid { Uuid::new_v5( diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs index 6ddee7f820..0e8d05b971 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs @@ -4,9 +4,9 @@ use std::collections::BTreeSet; use chrono::{DateTime, Utc}; -use crate::scheduler::_helpers::ticks::{action, requested_run, tick_id}; +use crate::scheduler::_helpers::ticks::catch_up_actions; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; -use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickPlan, TickState}; +use crate::scheduler::models::{TickPlan, TickState}; use crate::store::errors::ControlError; /// Return facts to append and the cursor to commit atomically with them. @@ -34,33 +34,7 @@ pub fn plan_ticks(state: &TickState, now: DateTime) -> Result = match (covering_run, state.catch_up) { - (Some(run_id), _) => due - .iter() - .map(|time| action(state.trigger_id, *time, Some(run_id))) - .collect(), - (None, CatchUp::All { max_runs }) => due - .iter() - .take(max_runs.get()) - .map(|time| action(state.trigger_id, *time, None)) - .collect(), - (None, CatchUp::Latest) => { - let latest = due.last().copied(); - let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); - due.iter() - .map(|time| { - action( - state.trigger_id, - *time, - (Some(*time) != latest).then_some(run_id).flatten(), - ) - }) - .collect() - } - }; + let actions = catch_up_actions(state, &due); let last_tick_for = actions .iter() .rev() diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs index eb7b841932..e05f14da2d 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -1,10 +1,10 @@ //! Immutable body inserts shared by snapshot publication. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use uuid::Uuid; -use crate::snapshots::models::PreparedNode; +use crate::snapshots::models::{PreparedNode, SnapshotChange}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -32,3 +32,38 @@ pub(crate) async fn store_nodes( } Ok(transaction) } + +pub(crate) async fn store_changes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + previous: &BTreeMap, + hashes: &BTreeMap, +) -> Result { + let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); + let mut sequence = 0; + for key in keys { + let before = previous.get(key); + let after = hashes.get(key); + if before == after { + continue; + } + let change = SnapshotChange { + change_kind: match (before, after) { + (None, _) => "added", + (_, None) => "removed", + _ => "changed", + } + .into(), + subject_key: key.clone(), + before_hash: before.cloned(), + after_hash: after.cloned(), + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ + VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(snapshot_id), Parameter::Integer(sequence), + Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), + Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; + sequence += 1; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs index 0861207ebc..c34cf82ab9 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -1,6 +1,6 @@ //! Publish a complete generation and its current pointer in one transaction. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use serde_json::Value; use uuid::Uuid; @@ -8,8 +8,8 @@ use uuid::Uuid; use crate::declarations::main::publish_definitions::publish_definitions; use crate::declarations::main::snapshot_definitions::snapshot_definitions; use crate::snapshots::_helpers::preparation::prepare_snapshot; -use crate::snapshots::_helpers::publication::store_nodes; -use crate::snapshots::models::{PreparedSnapshot, SnapshotChange, SnapshotInput, SnapshotRecord}; +use crate::snapshots::_helpers::publication::{store_changes, store_nodes}; +use crate::snapshots::models::{PreparedSnapshot, SnapshotInput, SnapshotRecord}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -81,32 +81,7 @@ pub async fn publish_snapshot( "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; } - let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); - let mut sequence = 0; - for key in keys { - let before = previous.get(key); - let after = hashes.get(key); - if before == after { - continue; - } - let change = SnapshotChange { - change_kind: match (before, after) { - (None, _) => "added", - (_, None) => "removed", - _ => "changed", - } - .into(), - subject_key: key.clone(), - before_hash: before.cloned(), - after_hash: after.cloned(), - }; - (transaction, _) = query(transaction, - "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ - VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(record.snapshot_id), Parameter::Integer(sequence), - Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), - Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; - sequence += 1; - } + transaction = store_changes(transaction, record.snapshot_id, &previous, &hashes).await?; (transaction, _) = query(transaction, "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 53420e77dc..a29a62c75f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -1,9 +1,37 @@ //! One schema template, rendered only at the database type boundary. +use semver::Version; + +use crate::store::errors::ControlError; use crate::store::models::ControlStore; +use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +pub(crate) fn verify_migrations(rows: &[Row], digest: &str) -> Result { + let current = Version::parse(env!("CARGO_PKG_VERSION")) + .map_err(|error| ControlError::Schema(error.to_string()))?; + for row in rows { + let minimum = row + .get("min_server_version") + .and_then(|value| value.as_str()) + .ok_or_else(|| { + ControlError::Schema("migration lacks a minimum server version".into()) + })?; + let minimum = + Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; + if row.get("version").and_then(|value| value.as_i64()) != Some(1) + || minimum > current + || row.get("checksum").and_then(|value| value.as_str()) != Some(digest) + { + return Err(ControlError::Schema( + "unknown migration, checksum mismatch, or minimum server version not met".into(), + )); + } + } + Ok(current) +} + pub(crate) fn render(store: &ControlStore, sql: &str) -> String { let postgres = matches!(store, ControlStore::Postgres(_)); sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index b5a38030f1..fca5fe59c0 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -1,10 +1,9 @@ //! Forward-only migration under the same lock used by all other control-plane writes. -use semver::Version; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::store::_helpers::migration::{SCHEMA, render}; +use crate::store::_helpers::migration::{SCHEMA, render, verify_migrations}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -24,28 +23,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { vec![], ) .await?; - let current = Version::parse(env!("CARGO_PKG_VERSION")) - .map_err(|error| ControlError::Schema(error.to_string()))?; let recall_statements = recall_schema(store); let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - for row in &rows { - let minimum = row - .get("min_server_version") - .and_then(|value| value.as_str()) - .ok_or_else(|| { - ControlError::Schema("migration lacks a minimum server version".into()) - })?; - let minimum = - Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; - if row.get("version").and_then(|value| value.as_i64()) != Some(1) - || minimum > current - || row.get("checksum").and_then(|value| value.as_str()) != Some(digest.as_str()) - { - return Err(ControlError::Schema( - "unknown migration, checksum mismatch, or minimum server version not met".into(), - )); - } - } + let current = verify_migrations(&rows, &digest)?; if rows.is_empty() { for statement in SCHEMA .split(';') From 6712115f7f3c5255c3777ee02033c5592f1d5ab2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:44:47 +0000 Subject: [PATCH 008/138] feat: observe platform health independently of scheduler leadership --- .../src/native_commands/main/_run_control.rs | 14 ++ .../tests/native_commands/helpers.rs | 11 +- .../tests/native_commands/test_control.rs | 10 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/mod.rs | 2 + .../src/platform/_helpers/observations.rs | 57 ++++++ .../src/platform/_helpers/watchdog.rs | 123 +++++++++++++ .../src/platform/main/mod.rs | 1 + .../src/platform/main/run_watchdog.rs | 25 +++ .../src/platform/mod.rs | 3 + .../src/platform/models.rs | 50 ++++++ .../src/runs/_helpers/events.rs | 52 ++---- .../src/store/main/append_control_event.rs | 52 ++++++ .../src/store/main/mod.rs | 1 + .../src/store/models.rs | 12 ++ .../sqlbuild-orchestration/tests/platform.rs | 12 ++ .../tests/platform/helpers.rs | 107 +++++++++++ .../tests/platform/test_types.rs | 26 +++ .../tests/platform/test_watchdog.rs | 166 ++++++++++++++++++ 19 files changed, 677 insertions(+), 48 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/append_control_event.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 7de55c4455..06f74ae5df 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,6 +1,7 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -53,6 +54,19 @@ pub(crate) async fn run_control( listener, stopped(shutdown.clone()), )); + drop(write_output(&[( + Stream::Stderr, + "Starting platform watchdog...\n", + )])); + let watchdog_store = store.clone(); + let watchdog_shutdown = shutdown.clone(); + tasks.spawn(async move { + run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await + }); + drop(write_output(&[( + Stream::Stderr, + "Platform watchdog started\n", + )])); } if standalone || !flag(namespace, "control_no_scheduler") { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 5e4ae4bfde..258d9c319d 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -65,16 +65,19 @@ pub(crate) async fn api_ready(control: &mut TestControl) -> String { .expect("native API becomes ready") } -pub(crate) fn leader_count(control: &TestControl) -> Option { +pub(crate) fn health_counts(control: &TestControl) -> Option<(i64, i64)> { let connection = rusqlite::Connection::open_with_flags( control.directory.path().join(".sqlbuild/control.sqlite3"), rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, ) .ok()?; connection - .query_row("SELECT COUNT(*) FROM sqb_scheduler_lease", [], |row| { - row.get(0) - }) + .query_row( + "SELECT (SELECT COUNT(*) FROM sqb_scheduler_lease), \ + (SELECT COUNT(*) FROM recall_events WHERE event_type='platform_heartbeat')", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) .ok() } diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index de16cd7367..b3394df6e3 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,7 +1,7 @@ use std::process::Command; use std::time::Duration; -use crate::helpers::{api_ready, isolated_binary, leader_count, start_control}; +use crate::helpers::{api_ready, health_counts, isolated_binary, start_control}; #[test] fn given_a_standalone_binary_when_printing_control_help_or_openapi_then_python_and_database_access_are_not_required() @@ -66,12 +66,14 @@ async fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then .expect("protected native API"); assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); tokio::time::timeout(Duration::from_secs(5), async { - while leader_count(&control) != Some(i64::from(scheduler)) { + while !health_counts(&control).is_some_and(|(leaders, heartbeats)| { + leaders == i64::from(scheduler) && heartbeats > 0 + }) { tokio::time::sleep(Duration::from_millis(20)).await; } }) .await - .expect("embedded scheduler enabled exactly when requested"); + .expect("watchdog records health with or without the optional scheduler"); #[cfg(unix)] crate::helpers::terminate(&mut control).await; } @@ -82,7 +84,7 @@ async fn given_a_standalone_scheduler_when_started_without_python_then_it_migrat { let mut control = start_control(&["scheduler"]); tokio::time::timeout(Duration::from_secs(5), async { - while leader_count(&control) != Some(1) { + while health_counts(&control).map(|(leaders, _)| leaders) != Some(1) { assert!( control .child diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 89689306a8..2546686e46 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -5,6 +5,7 @@ pub mod api; pub mod credentials; pub mod declarations; +pub mod platform; pub mod runs; pub mod scheduler; pub mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs new file mode 100644 index 0000000000..b604790017 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod observations; +pub(crate) mod watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs new file mode 100644 index 0000000000..8f5779d7fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs @@ -0,0 +1,57 @@ +//! Replicas share a deterministic observation key and a serialized append transaction. + +use chrono::{DurationRound, TimeDelta}; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::PlatformEvent; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn observe( + transaction: ControlTransaction, + installation_id: Uuid, + event: PlatformEvent, +) -> Result { + let occurred_at = transaction + .now + .duration_trunc(TimeDelta::seconds(30)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let payload = json!({"platform": event}); + let identity = serde_json::to_vec(&(occurred_at, &event)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&installation_id, &identity); + let (transaction, previous) = query( + transaction, + "SELECT 1 AS present FROM recall_events WHERE event_id=$1 AND occurred_at=$2 LIMIT 1", + vec![ + Parameter::Text(event_id.to_string()), + Parameter::Time(occurred_at), + ], + ) + .await?; + if !previous.is_empty() { + return Ok(transaction); + } + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("platform/{installation_id}/{}", occurred_at.date_naive()), + event_type: event.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "platform observation conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs new file mode 100644 index 0000000000..1757b07c66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -0,0 +1,123 @@ +//! Health scanning releases the writer lock between bounded trigger pages. + +use chrono::TimeDelta; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::declarations::models::Schedule; +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn scan_health( + store: &ControlStore, + replica_id: Uuid, + shutdown: &watch::Receiver, +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT i.installation_id, CASE WHEN l.expires_at>$1 THEN 1 ELSE 0 END AS leader \ + FROM sqb_installation i LEFT JOIN sqb_scheduler_lease l ON l.name='scheduler' WHERE i.singleton=1", + vec![Parameter::Time(now)]).await?; + let health: InstallationHealth = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("control store has no installation".into()))?, + )?; + let mut transaction = transaction; + if health.leader == 0 { + transaction = observe( + transaction, + health.installation_id, + PlatformEvent::SchedulerNoLeader, + ) + .await?; + } + commit(transaction).await?; + let mut after = None; + while !*shutdown.borrow() { + after = scan_trigger_page(store, health.installation_id, after).await?; + if after.is_none() { + break; + } + tokio::task::yield_now().await; + } + if *shutdown.borrow() { + return Ok(()); + } + let transaction = begin(store, TransactionMode::Write).await?; + commit( + observe( + transaction, + health.installation_id, + PlatformEvent::Heartbeat { replica_id }, + ) + .await?, + ) + .await?; + Ok(()) +} + +async fn scan_trigger_page( + store: &ControlStore, + installation_id: Uuid, + after: Option, +) -> Result, ControlError> { + let cursor = after.map_or("", |_| "AND t.trigger_id>$1"); + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT t.trigger_id, t.definition_json, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + {cursor} ORDER BY t.trigger_id LIMIT 100" + ), + after.into_iter().map(Parameter::Id).collect(), + ) + .await?; + let triggers: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = triggers.last().map(|trigger| trigger.trigger_id); + for trigger in triggers { + if let Some(scheduled_for) = trigger.next_due_at + && scheduled_for < now - TimeDelta::seconds(30) + { + transaction = observe( + transaction, + installation_id, + PlatformEvent::TickOverdue { + trigger_id: trigger.trigger_id, + scheduled_for, + }, + ) + .await?; + } + if let Schedule::Calendar { times, refresh_at } = trigger.definition_json.schedule + && times + .last() + .is_none_or(|last| *last <= now + TimeDelta::days(1)) + { + transaction = observe( + transaction, + installation_id, + PlatformEvent::CalendarHorizonLow { + trigger_id: trigger.trigger_id, + refresh_at, + horizon_end: times.last().copied(), + }, + ) + .await?; + } + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs new file mode 100644 index 0000000000..bb879e253f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -0,0 +1 @@ +pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs b/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs new file mode 100644 index 0000000000..d8a72dc0fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs @@ -0,0 +1,25 @@ +//! Every server observes health independently; scheduler leadership is not required. + +use std::time::Duration; + +use tokio::sync::watch; +use uuid::Uuid; + +use crate::platform::_helpers::watchdog::scan_health; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_watchdog( + store: &ControlStore, + replica_id: Uuid, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + while !*shutdown.borrow() { + scan_health(store, replica_id, &shutdown).await?; + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(30)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs new file mode 100644 index 0000000000..0f3bb0325c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -0,0 +1,50 @@ +//! Platform health is a typed stream of observations, independent of user code. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::TriggerDefinition; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PlatformEvent { + Heartbeat { + replica_id: Uuid, + }, + SchedulerNoLeader, + TickOverdue { + trigger_id: Uuid, + scheduled_for: DateTime, + }, + CalendarHorizonLow { + trigger_id: Uuid, + refresh_at: DateTime, + horizon_end: Option>, + }, +} + +impl PlatformEvent { + #[must_use] + pub const fn event_type(&self) -> &'static str { + match self { + Self::Heartbeat { .. } => "platform_heartbeat", + Self::SchedulerNoLeader => "platform_scheduler_no_leader", + Self::TickOverdue { .. } => "platform_tick_overdue", + Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", + } + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct InstallationHealth { + pub(crate) installation_id: Uuid, + pub(crate) leader: i64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) definition_json: TriggerDefinition, + pub(crate) next_due_at: Option>, +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs index 7ef23c279b..cb7b38073c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs @@ -1,15 +1,11 @@ //! All orchestration transitions share Recall's invocation sequence and transaction. -use chrono::SecondsFormat; -use serde_json::{Map, Value, json}; -use sqlbuild_recall::records::main::event_object_row::event_object_row; -use sqlbuild_recall::records::models::RowEnvelope; +use serde_json::Value; use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::append_recall::append_recall; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; pub(crate) async fn append_run_event( transaction: ControlTransaction, @@ -17,40 +13,16 @@ pub(crate) async fn append_run_event( kind: &str, payload: Value, ) -> Result<(ControlTransaction, i64), ControlError> { - let invocation_id = format!("scheduler/{run_id}"); - let timestamp = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); - let (transaction, rows) = query(transaction, - "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ - WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", - vec![Parameter::Text(invocation_id.clone())]).await?; - let previous = rows.first(); - let sequence = previous - .and_then(|row| row.get("last_sequence")) - .and_then(Value::as_i64) - .map_or(Some(0), |value| value.checked_add(1)) - .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; - let envelope = RowEnvelope { - invocation_id, - invocation_started_at: previous - .and_then(|row| row.get("started_at")) - .and_then(Value::as_str) - .unwrap_or(×tamp) - .into(), - producer_version: env!("CARGO_PKG_VERSION").into(), + let event = ControlEvent { + event_id, + invocation_id: format!("scheduler/{run_id}"), + event_type: kind, + producer: "scheduler", + occurred_at: transaction.now, + run_id: Some(run_id), + payload, }; - let event = Map::from_iter([ - ("event_id".into(), json!(event_id)), - ("event_type".into(), json!(kind)), - ("schema_version".into(), json!(1)), - ("occurred_at".into(), json!(timestamp)), - ("producer".into(), json!("scheduler")), - ("invocation_sequence".into(), json!(sequence)), - ("run_id".into(), json!(run_id)), - ("payload".into(), payload), - ]); - let row = event_object_row(&envelope, &event) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, outcome) = append_recall(transaction, &row).await?; + let (transaction, outcome) = append_control_event(transaction, event).await?; if outcome.conflict.is_some() { return Err(ControlError::Invalid( "run event conflicts with a durable fact".into(), diff --git a/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs new file mode 100644 index 0000000000..3143159449 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs @@ -0,0 +1,52 @@ +//! Sequence scheduler and platform facts inside the caller's serialized transaction. + +use chrono::SecondsFormat; +use serde_json::{Map, Value, json}; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::RowEnvelope; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter, RecallAppendOutcome}; + +pub(crate) async fn append_control_event( + transaction: ControlTransaction, + event: ControlEvent<'_>, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let timestamp = event + .occurred_at + .to_rfc3339_opts(SecondsFormat::Micros, true); + let (transaction, rows) = query(transaction, + "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ + WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(event.invocation_id.clone())]).await?; + let previous = rows.first(); + let sequence = previous + .and_then(|row| row.get("last_sequence")) + .and_then(Value::as_i64) + .map_or(Some(0), |value| value.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; + let envelope = RowEnvelope { + invocation_id: event.invocation_id, + invocation_started_at: previous + .and_then(|row| row.get("started_at")) + .and_then(Value::as_str) + .unwrap_or(×tamp) + .into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + }; + let object = Map::from_iter([ + ("event_id".into(), json!(event.event_id)), + ("event_type".into(), json!(event.event_type)), + ("schema_version".into(), json!(1)), + ("occurred_at".into(), json!(timestamp)), + ("producer".into(), json!(event.producer)), + ("invocation_sequence".into(), json!(sequence)), + ("run_id".into(), json!(event.run_id)), + ("payload".into(), event.payload), + ]); + let row = event_object_row(&envelope, &object) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + append_recall(transaction, &row).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index ba1903c103..d22e12f725 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod append_control_event; pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index 3c097f41bd..414bca79cd 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -22,6 +22,18 @@ pub struct RecallAppendOutcome { pub conflict: Option, } +/// A control-plane fact shares Recall's durable sequencing and conflict handling. +#[derive(Debug)] +pub(crate) struct ControlEvent<'a> { + pub(crate) event_id: Uuid, + pub(crate) invocation_id: String, + pub(crate) event_type: &'a str, + pub(crate) producer: &'static str, + pub(crate) occurred_at: DateTime, + pub(crate) run_id: Option, + pub(crate) payload: serde_json::Value, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum TransactionMode { Read, diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs new file mode 100644 index 0000000000..673cc30aef --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform.rs @@ -0,0 +1,12 @@ +#[path = "platform/helpers.rs"] +mod helpers; +#[path = "store/replica_helpers.rs"] +mod replica_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "platform/test_types.rs"] +mod test_types; +#[path = "platform/test_watchdog.rs"] +mod test_watchdog; diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs new file mode 100644 index 0000000000..c2ea45fa05 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -0,0 +1,107 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use sqlbuild_orchestration::platform::models::PlatformEvent; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::task::JoinHandle; +use uuid::Uuid; + +use crate::test_types::Observation; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let future = Utc::now() + TimeDelta::days(10); + let mut triggers = BTreeMap::new(); + for index in 0..101 { + triggers.insert( + format!("daily_orders_{index}"), + json!({ + "description": "Refresh orders", "job_key": "orders", + "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", future], + "refresh_at": "2020-01-01T00:00:00Z"} + }), + ); + } + for (name, status) in [("expired_orders", "running"), ("paused_orders", "paused")] { + triggers.insert(name.into(), json!({ + "description": "Refresh orders", "job_key": "orders", "default_status": status, + "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2020-01-01T00:00:00Z"} + })); + } + triggers.insert( + "future_orders".into(), + json!({ + "description": "Future orders refresh", "job_key": "orders", + "schedule": {"kind": "calendar", "times": [future], "refresh_at": future} + }), + ); + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-watchdog-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ("triggers".into(), json!(triggers)), + ]), + } +} + +pub(crate) async fn wait_for_scans( + store: &ControlStore, + replicas: &BTreeSet, + tasks: &[JoinHandle>], +) -> Vec { + tokio::time::timeout(Duration::from_secs(10), async { + let mut cursor = 0; + let mut observations = Vec::new(); + let mut scanned = BTreeSet::new(); + while &scanned != replicas { + assert!( + tasks.iter().all(|task| !task.is_finished()), + "watchdog failed before scanning" + ); + for row in read_recall_events(store, cursor, 1000) + .await + .expect("read health facts") + { + let observation: Observation = + serde_json::from_value(json!(row)).expect("typed platform event"); + cursor = observation.storage_id; + scanned.extend(match observation.payload.platform { + PlatformEvent::Heartbeat { replica_id } => Some(replica_id), + _ => None, + }); + observations.push(observation); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + observations + }) + .await + .expect("both replicas finish a paged scan without a scheduler") +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs new file mode 100644 index 0000000000..1f1fec3985 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -0,0 +1,26 @@ +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use sqlbuild_orchestration::platform::models::PlatformEvent; + +use crate::store_test_types::Backend; + +pub(crate) struct WatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) take_lease: bool, + pub(crate) expected_no_leader: bool, + pub(crate) expected_overdue: usize, + pub(crate) expected_low_horizons: usize, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct Observation { + pub(crate) storage_id: i64, + pub(crate) occurred_at: DateTime, + pub(crate) payload: PlatformPayload, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct PlatformPayload { + pub(crate) platform: PlatformEvent, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs new file mode 100644 index 0000000000..26e40396d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -0,0 +1,166 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; +use sqlbuild_orchestration::platform::models::PlatformEvent; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{snapshot, wait_for_scans}; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::WatchdogTestCase; + +#[test] +fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_durable_deduplicated_and_independent_of_leadership() + { + let test_cases = [ + WatchdogTestCase { + description: "SQLite without a leader", + backend: Backend::Sqlite, + take_lease: false, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL without a leader", + backend: Backend::Postgres, + take_lease: false, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "SQLite with a frozen leader", + backend: Backend::Sqlite, + take_lease: true, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL with a frozen leader", + backend: Backend::Postgres, + take_lease: true, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate health store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input + .documents + .get_mut("triggers") + .expect("trigger document")["removed_orders"] = + input.documents["triggers"]["expired_orders"].clone(); + publish_snapshot(store, &input) + .await + .expect("old definition"); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document") + .as_object_mut() + .expect("trigger map") + .remove("removed_orders"); + publish_snapshot(store, &input) + .await + .expect("current definition"); + if test_case.take_lease { + acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("leader"); + } + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { run_watchdog(&store, id, shutdown).await }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let overdue: BTreeSet<_> = observations + .iter() + .filter_map(|row| match row.payload.platform { + PlatformEvent::TickOverdue { trigger_id, .. } => Some(trigger_id), + _ => None, + }) + .collect(); + assert_eq!( + overdue.len(), + test_case.expected_overdue, + "{}", + test_case.description + ); + let horizons: BTreeSet<_> = observations + .iter() + .filter_map(|row| match row.payload.platform { + PlatformEvent::CalendarHorizonLow { trigger_id, .. } => Some(trigger_id), + _ => None, + }) + .collect(); + assert_eq!( + horizons.len(), + test_case.expected_low_horizons, + "paused and removed calendars are excluded; refresh cadence alone is not a low horizon" + ); + assert_eq!( + observations + .iter() + .any(|row| matches!(row.payload.platform, PlatformEvent::SchedulerNoLeader)), + test_case.expected_no_leader + ); + let logical_keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("event identity"), + ) + }) + .collect(); + assert_eq!( + logical_keys.len(), + observations.len(), + "replicas do not duplicate an observation within its window" + ); + assert!( + observations + .iter() + .all(|row| row.occurred_at.timestamp().rem_euclid(30) == 0) + ); + stop.send(true).expect("shutdown both replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join watchdog") + .expect("watchdog stopped"); + } + close_store(second).await; + close_test_store(fixture).await; + }); + } +} From 18ecdfa09f728d1f633fcfb403fc0da1882b1ecb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:01:11 +0000 Subject: [PATCH 009/138] test: align backend integration contracts with domain ownership --- .../tests/native_commands/helpers.rs | 71 ++++-- .../tests/native_commands/test_control.rs | 240 ++++++++++++------ .../tests/native_commands/test_types.rs | 20 ++ .../tests/api/helpers.rs | 19 +- .../tests/credentials.rs | 12 +- .../tests/credentials/helpers.rs | 21 ++ .../tests/credentials/test_contract.rs | 53 ++-- .../tests/credentials/test_types.rs | 8 + .../sqlbuild-orchestration/tests/platform.rs | 2 +- .../tests/platform/helpers.rs | 30 ++- .../tests/platform/test_types.rs | 2 +- .../tests/platform/test_watchdog.rs | 27 +- crates/sqlbuild-orchestration/tests/recall.rs | 14 - crates/sqlbuild-orchestration/tests/runs.rs | 10 +- .../helpers.rs} | 2 + .../tests/runs/test_faults.rs | 6 +- .../tests/runs/test_lifecycle.rs | 6 +- .../tests/runs/test_types.rs | 2 +- .../sqlbuild-orchestration/tests/scheduler.rs | 10 +- .../tests/{ => scheduler}/ticks/helpers.rs | 18 +- .../{ => scheduler}/ticks/test_contract.rs | 6 +- .../tests/{ => scheduler}/ticks/test_loop.rs | 6 +- .../tests/scheduler/ticks/test_ticks.rs | 10 + .../{ => scheduler}/ticks/test_triggers.rs | 8 +- .../tests/{ => scheduler}/ticks/test_types.rs | 4 +- .../sqlbuild-orchestration/tests/snapshots.rs | 2 +- crates/sqlbuild-orchestration/tests/store.rs | 16 ++ .../faults/helpers.rs} | 8 +- .../{fault_types.rs => faults/test_types.rs} | 4 +- .../tests/store/lease_helpers.rs | 24 -- .../tests/store/leases/helpers.rs | 24 ++ .../tests/{ => store}/recall/helpers.rs | 2 + .../tests/{ => store}/recall/test_contract.rs | 2 + .../tests/{ => store}/recall/test_types.rs | 2 + .../tests/{ => store}/recall/test_views.rs | 2 + .../tests/store/replica_helpers.rs | 15 -- .../tests/store/replicas/helpers.rs | 20 ++ crates/sqlbuild-orchestration/tests/ticks.rs | 22 -- 38 files changed, 474 insertions(+), 276 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/credentials/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_types.rs delete mode 100644 crates/sqlbuild-orchestration/tests/recall.rs rename crates/sqlbuild-orchestration/tests/runs/{terminal_helpers.rs => termination/helpers.rs} (93%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/helpers.rs (96%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_contract.rs (83%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_loop.rs (96%) create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_triggers.rs (96%) rename crates/sqlbuild-orchestration/tests/{ => scheduler}/ticks/test_types.rs (78%) create mode 100644 crates/sqlbuild-orchestration/tests/store.rs rename crates/sqlbuild-orchestration/tests/{ticks/fault_helpers.rs => store/faults/helpers.rs} (92%) rename crates/sqlbuild-orchestration/tests/store/{fault_types.rs => faults/test_types.rs} (68%) delete mode 100644 crates/sqlbuild-orchestration/tests/store/lease_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/leases/helpers.rs rename crates/sqlbuild-orchestration/tests/{ => store}/recall/helpers.rs (98%) rename crates/sqlbuild-orchestration/tests/{ => store}/recall/test_contract.rs (93%) rename crates/sqlbuild-orchestration/tests/{ => store}/recall/test_types.rs (84%) rename crates/sqlbuild-orchestration/tests/{ => store}/recall/test_views.rs (99%) delete mode 100644 crates/sqlbuild-orchestration/tests/store/replica_helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs delete mode 100644 crates/sqlbuild-orchestration/tests/ticks.rs diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 258d9c319d..606246903b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -2,10 +2,10 @@ use std::path::Path; use std::process::{Command, Stdio}; use std::time::Duration; -use crate::test_types::TestControl; +use crate::test_types::{ServiceTestCase, TestControl}; pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { - let binary = directory.join(if cfg!(windows) { "sqb.exe" } else { "sqb" }); + let binary = directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX)); std::fs::copy(env!("CARGO_BIN_EXE_sqb"), &binary) .expect("copy standalone binary without Python CLI"); binary @@ -37,29 +37,33 @@ pub(crate) async fn api_ready(control: &mut TestControl) -> String { .build() .expect("HTTP client"); tokio::time::timeout(Duration::from_secs(10), async { - loop { + for _ in 0..500 { assert!( control.child.try_wait().expect("child status").is_none(), "control process exited: {}", std::fs::read_to_string(&control.stderr).unwrap_or_default() ); + tokio::time::sleep(Duration::from_millis(20)).await; let log = std::fs::read_to_string(&control.stderr).expect("read control log"); - if let Some(address) = log + let Some(address) = log .lines() .find_map(|line| line.strip_prefix("Control API ready: ")) - { - let url = format!("http://{address}"); - if client - .get(format!("{url}/api/openapi.json")) - .send() - .await - .is_ok_and(|response| response.status().is_success()) - { - return url; - } - } - tokio::time::sleep(Duration::from_millis(20)).await; + else { + continue; + }; + let url = format!("http://{address}"); + let Ok(response) = client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return url; } + panic!("native API exhausted readiness attempts"); }) .await .expect("native API becomes ready") @@ -81,6 +85,31 @@ pub(crate) fn health_counts(control: &TestControl) -> Option<(i64, i64)> { .ok() } +pub(crate) async fn wait_for_health( + control: &mut TestControl, + case: &ServiceTestCase, +) -> (i64, i64) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + assert!( + control.child.try_wait().expect("service status").is_none(), + "service exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + let Some(counts) = health_counts(control).filter(|(leaders, heartbeats)| { + *leaders == case.expected_leaders && *heartbeats >= case.expected_heartbeats + }) else { + continue; + }; + return counts; + } + panic!("control service exhausted health readiness attempts"); + }) + .await + .expect("native control service publishes its health state") +} + #[cfg(unix)] pub(crate) async fn terminate(control: &mut TestControl) { let status = Command::new("kill") @@ -89,12 +118,14 @@ pub(crate) async fn terminate(control: &mut TestControl) { .expect("send SIGTERM"); assert!(status.success()); let status = tokio::time::timeout(Duration::from_secs(5), async { - loop { - if let Some(status) = control.child.try_wait().expect("child exit") { - return status; - } + for _ in 0..250 { tokio::time::sleep(Duration::from_millis(20)).await; + let Some(status) = control.child.try_wait().expect("child exit") else { + continue; + }; + return status; } + panic!("control roles exhausted shutdown attempts"); }) .await .expect("control roles stop promptly"); diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index b3394df6e3..85a2e4e44c 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,102 +1,188 @@ use std::process::Command; -use std::time::Duration; -use crate::helpers::{api_ready, health_counts, isolated_binary, start_control}; +use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; +use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase}; #[test] -fn given_a_standalone_binary_when_printing_control_help_or_openapi_then_python_and_database_access_are_not_required() +fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() { - let directory = tempfile::tempdir().expect("isolated CLI directory"); - let binary = isolated_binary(directory.path()); - for arguments in [ - ["server", "--help"], - ["scheduler", "--help"], - ["server", "--openapi"], - ] { + let test_cases = [ + CommandTestCase { + description: "native server help", + arguments: &["server", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "native scheduler help", + arguments: &["scheduler", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "invalid port is rejected before store initialization", + arguments: &["server", "--port", "65536"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control service failed: invalid control-plane data: --port must be between 0 and 65535\n", + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); let output = Command::new(&binary) - .args(arguments) + .args(test_case.arguments) .current_dir(directory.path()) .env("PATH", "") .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") .output() .expect("standalone native command"); - assert!( + assert_eq!( output.status.success(), + test_case.expected_success, + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + test_case.expected_stderr, + "{}", + test_case.description + ); + assert!( + String::from_utf8_lossy(&output.stdout).starts_with(test_case.expected_stdout_prefix), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), "{}", - String::from_utf8_lossy(&output.stderr) + test_case.description ); - assert!(output.stderr.is_empty()); - if arguments[1] == "--openapi" { - let document: serde_json::Value = - serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); - assert!(document["paths"]["/api/runs/{run_id}/events"].is_object()); - } else { - assert!(String::from_utf8_lossy(&output.stdout).starts_with("usage: sqb ")); - } } - assert!(!directory.path().join(".sqlbuild").exists()); - let output = Command::new(&binary) - .args(["server", "--port", "65536"]) - .current_dir(directory.path()) - .env("PATH", "") - .output() - .expect("invalid port"); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("--port must be between 0 and 65535")); - assert!( - !directory.path().join(".sqlbuild").exists(), - "invalid input cannot initialize a database" - ); } -#[tokio::test] -async fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() +#[test] +fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_machine_readable_without_python_or_a_database() { - for scheduler in [false, true] { - let mut arguments = vec!["server", "--port", "0"]; - if !scheduler { - arguments.push("--no-scheduler"); - } - let mut control = start_control(&arguments); - let url = api_ready(&mut control).await; - let response = reqwest::Client::new() - .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) - .send() - .await - .expect("protected native API"); - assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); - tokio::time::timeout(Duration::from_secs(5), async { - while !health_counts(&control).is_some_and(|(leaders, heartbeats)| { - leaders == i64::from(scheduler) && heartbeats > 0 - }) { - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .expect("watchdog records health with or without the optional scheduler"); - #[cfg(unix)] - crate::helpers::terminate(&mut control).await; + let test_cases = [OpenApiTestCase { + description: "worker lifecycle endpoint", + expected_path: "/api/runs/{run_id}/events", + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let output = Command::new(isolated_binary(directory.path())) + .args(["server", "--openapi"]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone OpenAPI"); + assert!(output.status.success(), "{}", test_case.description); + assert!(output.stderr.is_empty(), "{}", test_case.description); + let document: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); + assert!( + document["paths"][test_case.expected_path].is_object(), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), + "{}", + test_case.description + ); } } -#[tokio::test] -async fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() +#[test] +fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() { - let mut control = start_control(&["scheduler"]); - tokio::time::timeout(Duration::from_secs(5), async { - while health_counts(&control).map(|(leaders, _)| leaders) != Some(1) { + let test_cases = [ + ServiceTestCase { + description: "server without scheduling still observes platform health", + arguments: &["server", "--port", "0", "--no-scheduler"], + expected_leaders: 0, + expected_heartbeats: 1, + }, + ServiceTestCase { + description: "server with embedded scheduling", + arguments: &["server", "--port", "0"], + expected_leaders: 1, + expected_heartbeats: 1, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let url = api_ready(&mut control).await; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("bounded HTTP requests"); + let response = client + .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) + .send() + .await + .expect("protected native API"); + assert_eq!( + response.status(), + reqwest::StatusCode::UNAUTHORIZED, + "{}", + test_case.description + ); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); assert!( - control - .child - .try_wait() - .expect("scheduler status") - .is_none() + heartbeats >= test_case.expected_heartbeats, + "{}", + test_case.description ); - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .expect("standalone scheduler acquires leadership"); - #[cfg(unix)] - crate::helpers::terminate(&mut control).await; + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() + { + let test_cases = [ServiceTestCase { + description: "standalone scheduler takes a lease without starting a server watchdog", + arguments: &["scheduler"], + expected_leaders: 1, + expected_heartbeats: 0, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); + assert_eq!( + heartbeats, test_case.expected_heartbeats, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } } diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 35842e7f61..15974f0788 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -1,6 +1,26 @@ use std::path::PathBuf; use std::process::Child; +pub(crate) struct CommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_success: bool, + pub(crate) expected_stdout_prefix: &'static str, + pub(crate) expected_stderr: &'static str, +} + +pub(crate) struct OpenApiTestCase { + pub(crate) description: &'static str, + pub(crate) expected_path: &'static str, +} + +pub(crate) struct ServiceTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_leaders: i64, + pub(crate) expected_heartbeats: i64, +} + pub(crate) struct TestControl { pub(crate) child: Child, pub(crate) directory: tempfile::TempDir, diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 395e1535f5..4364fba3a3 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -20,18 +20,21 @@ pub(crate) async fn start_api(store: &ControlStore) -> TestApi { .build() .expect("HTTP client"); tokio::time::timeout(Duration::from_secs(5), async { - loop { - if client + for _ in 0..250 { + assert!(!task.is_finished(), "API failed during startup"); + tokio::time::sleep(Duration::from_millis(20)).await; + let Ok(response) = client .get(format!("{url}/api/openapi.json")) .send() .await - .is_ok_and(|response| response.status().is_success()) - { - break; - } - assert!(!task.is_finished(), "API failed during startup"); - tokio::time::sleep(Duration::from_millis(20)).await; + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return; } + panic!("API exhausted readiness attempts"); }) .await .expect("API becomes ready"); diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs index c40545cc9e..b31553bc0d 100644 --- a/crates/sqlbuild-orchestration/tests/credentials.rs +++ b/crates/sqlbuild-orchestration/tests/credentials.rs @@ -1,6 +1,10 @@ -#[path = "ticks/fault_helpers.rs"] +#[path = "store/faults/helpers.rs"] mod fault_helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/faults/test_types.rs"] +mod fault_types; +#[path = "credentials/helpers.rs"] +mod helpers; +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "runs/helpers.rs"] mod run_helpers; @@ -8,9 +12,9 @@ mod run_helpers; mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; -#[path = "runs/terminal_helpers.rs"] +#[path = "runs/termination/helpers.rs"] mod terminal_helpers; #[path = "credentials/test_contract.rs"] mod test_contract; -#[path = "store/fault_types.rs"] +#[path = "credentials/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs new file mode 100644 index 0000000000..e01295e856 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -0,0 +1,21 @@ +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + let connection = rusqlite::Connection::open(path).expect("inspect durable tokens"); + return connection + .prepare("SELECT secret_hash FROM sqb_access_tokens") + .expect("token hashes") + .query_map([], |row| row.get(0)) + .expect("read token hashes") + .collect::>() + .expect("hashes"); + }; + sqlx_core::query_scalar::query_scalar("SELECT secret_hash FROM sqb_access_tokens") + .fetch_all(pool) + .await + .expect("read durable token hashes") +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 9b0ff859f2..4c39180fad 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -11,23 +11,35 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::stored_hashes; use crate::replica_helpers::second_store; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::terminal_helpers::finish; -use crate::test_types::TickFaultTestCase; +use crate::test_types::{CredentialTestCase, TransactionFault}; #[test] fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + CredentialTestCase { + description: "SQLite credential lifecycle and atomic audit", + backend: Backend::Sqlite, + expected_hash_length: 64, + }, + CredentialTestCase { + description: "PostgreSQL credential lifecycle and atomic audit", + backend: Backend::Postgres, + expected_hash_length: 64, + }, + ]; + for test_case in test_cases { run(async { - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -39,7 +51,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the request_id: Uuid::now_v7(), ip_address: Some("127.0.0.1".parse().expect("loopback")), }; - let fault = TickFaultTestCase { + let fault = TransactionFault { description: "security audit failure", table: "sqb_security_audit", operation: "INSERT", @@ -87,11 +99,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the !format!("{issued:?}").contains(&issued.bearer), "debug output redacts secrets" ); - let replacement = if issued.bearer.ends_with('0') { - "1" - } else { - "0" - }; + let replacement = ["0", "1"][usize::from(issued.bearer.ends_with('0'))]; let tampered = format!("{}{replacement}", &issued.bearer[..issued.bearer.len() - 1]); for invalid in [ @@ -141,29 +149,14 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .await .is_err() ); - let stored: Vec = match store { - ControlStore::Sqlite(path) => { - let connection = - rusqlite::Connection::open(path).expect("inspect durable tokens"); - connection - .prepare("SELECT secret_hash FROM sqb_access_tokens") - .expect("token hashes") - .query_map([], |row| row.get(0)) - .expect("read token hashes") - .collect::>() - .expect("hashes") - } - ControlStore::Postgres(pool) => sqlx_core::query_scalar::query_scalar( - "SELECT secret_hash FROM sqb_access_tokens", - ) - .fetch_all(pool) - .await - .expect("read durable token hashes"), - }; + let stored = stored_hashes(store).await; assert!( stored .iter() - .all(|hash| hash.len() == 64 && hash != &issued.bearer) + .all(|hash| hash.len() == test_case.expected_hash_length + && hash != &issued.bearer), + "{}", + test_case.description ); } for scope in [ diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs new file mode 100644 index 0000000000..077ee03ce9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -0,0 +1,8 @@ +pub(crate) use crate::fault_types::TransactionFault; +use crate::store_test_types::Backend; + +pub(crate) struct CredentialTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_hash_length: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs index 673cc30aef..7908cbf127 100644 --- a/crates/sqlbuild-orchestration/tests/platform.rs +++ b/crates/sqlbuild-orchestration/tests/platform.rs @@ -1,6 +1,6 @@ #[path = "platform/helpers.rs"] mod helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index c2ea45fa05..310870ecfb 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -80,27 +80,31 @@ pub(crate) async fn wait_for_scans( let mut cursor = 0; let mut observations = Vec::new(); let mut scanned = BTreeSet::new(); - while &scanned != replicas { + for _ in 0..1000 { + let Some(_) = replicas.difference(&scanned).next() else { + return observations; + }; assert!( tasks.iter().all(|task| !task.is_finished()), "watchdog failed before scanning" ); - for row in read_recall_events(store, cursor, 1000) + let batch: Vec = read_recall_events(store, cursor, 1000) .await .expect("read health facts") - { - let observation: Observation = - serde_json::from_value(json!(row)).expect("typed platform event"); - cursor = observation.storage_id; - scanned.extend(match observation.payload.platform { - PlatformEvent::Heartbeat { replica_id } => Some(replica_id), - _ => None, - }); - observations.push(observation); - } + .into_iter() + .map(|row| serde_json::from_value(json!(row)).expect("typed platform event")) + .collect(); + cursor = batch.last().map_or(cursor, |row| row.storage_id); + scanned.extend(batch.iter().filter_map(|row| { + let PlatformEvent::Heartbeat { replica_id } = row.payload.platform else { + return None; + }; + Some(replica_id) + })); + observations.extend(batch); tokio::time::sleep(Duration::from_millis(10)).await; } - observations + panic!("watchdog exhausted scan completion attempts"); }) .await .expect("both replicas finish a paged scan without a scheduler") diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 1f1fec3985..418d6a8cb3 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -7,7 +7,7 @@ use crate::store_test_types::Backend; pub(crate) struct WatchdogTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, - pub(crate) take_lease: bool, + pub(crate) leaders: usize, pub(crate) expected_no_leader: bool, pub(crate) expected_overdue: usize, pub(crate) expected_low_horizons: usize, diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 26e40396d0..6954c6bfe0 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -22,7 +22,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "SQLite without a leader", backend: Backend::Sqlite, - take_lease: false, + leaders: 0, expected_no_leader: true, expected_overdue: 102, expected_low_horizons: 1, @@ -30,7 +30,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "PostgreSQL without a leader", backend: Backend::Postgres, - take_lease: false, + leaders: 0, expected_no_leader: true, expected_overdue: 102, expected_low_horizons: 1, @@ -38,7 +38,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "SQLite with a frozen leader", backend: Backend::Sqlite, - take_lease: true, + leaders: 1, expected_no_leader: false, expected_overdue: 102, expected_low_horizons: 1, @@ -46,7 +46,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du WatchdogTestCase { description: "PostgreSQL with a frozen leader", backend: Backend::Postgres, - take_lease: true, + leaders: 1, expected_no_leader: false, expected_overdue: 102, expected_low_horizons: 1, @@ -82,7 +82,7 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du publish_snapshot(store, &input) .await .expect("current definition"); - if test_case.take_lease { + for _ in 0..test_case.leaders { acquire_lease(store, Uuid::now_v7()) .await .expect("acquire lease") @@ -103,9 +103,11 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du let observations = wait_for_scans(store, &replicas, &tasks).await; let overdue: BTreeSet<_> = observations .iter() - .filter_map(|row| match row.payload.platform { - PlatformEvent::TickOverdue { trigger_id, .. } => Some(trigger_id), - _ => None, + .filter_map(|row| { + let PlatformEvent::TickOverdue { trigger_id, .. } = row.payload.platform else { + return None; + }; + Some(trigger_id) }) .collect(); assert_eq!( @@ -116,9 +118,12 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du ); let horizons: BTreeSet<_> = observations .iter() - .filter_map(|row| match row.payload.platform { - PlatformEvent::CalendarHorizonLow { trigger_id, .. } => Some(trigger_id), - _ => None, + .filter_map(|row| { + let PlatformEvent::CalendarHorizonLow { trigger_id, .. } = row.payload.platform + else { + return None; + }; + Some(trigger_id) }) .collect(); assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/recall.rs b/crates/sqlbuild-orchestration/tests/recall.rs deleted file mode 100644 index 3261be8a4e..0000000000 --- a/crates/sqlbuild-orchestration/tests/recall.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "recall/helpers.rs"] -mod helpers; -#[path = "store/replica_helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "recall/test_contract.rs"] -mod test_contract; -#[path = "recall/test_types.rs"] -mod test_types; -#[path = "recall/test_views.rs"] -mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 10a2813d22..5b2f42488c 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -1,18 +1,18 @@ -#[path = "ticks/fault_helpers.rs"] +#[path = "store/faults/helpers.rs"] mod fault_helpers; -#[path = "store/fault_types.rs"] +#[path = "store/faults/test_types.rs"] mod fault_types; #[path = "runs/helpers.rs"] mod helpers; -#[path = "store/lease_helpers.rs"] +#[path = "store/leases/helpers.rs"] mod lease_helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; #[path = "store/test_types.rs"] mod store_test_types; -#[path = "runs/terminal_helpers.rs"] +#[path = "runs/termination/helpers.rs"] mod terminal_helpers; #[path = "runs/test_claims.rs"] mod test_claims; diff --git a/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs similarity index 93% rename from crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs rename to crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs index 68e8742d32..9d6c347e0b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/terminal_helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -1,3 +1,5 @@ +//! Terminal facts exercise claim release through the public run lifecycle. + use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 3332a4bd5c..6af9167fbb 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -19,7 +19,7 @@ use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{ClaimFailureTestCase, TickFaultTestCase}; +use crate::test_types::{ClaimFailureTestCase, TransactionFault}; #[test] fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() @@ -78,7 +78,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene ("recall_events", "INSERT"), ("sqb_runs", "UPDATE"), ] { - let point = TickFaultTestCase { + let point = TransactionFault { description: "claim publication failure", table, operation, @@ -106,7 +106,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene projected ); } - let expiry = TickFaultTestCase { + let expiry = TransactionFault { description: "claim fence expires before commit", table: "sqb_run_liveness", operation: "INSERT", diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index d61b50e82a..0f02f31093 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -19,7 +19,7 @@ use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TickFaultTestCase}; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; #[test] fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() @@ -156,7 +156,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .await .expect("events before node fault"); for table in ["sqb_run_nodes", "sqb_runs"] { - let point = TickFaultTestCase { + let point = TransactionFault { description: "node projection and counter failure", table, operation: "UPDATE", @@ -217,7 +217,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ ("sqb_write_claims", "DELETE"), ("sqb_run_liveness", "DELETE"), ] { - let point = TickFaultTestCase { + let point = TransactionFault { description: "terminal transaction failure", table, operation, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 61c94312d4..81471a4f67 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -2,7 +2,7 @@ use sqlbuild_orchestration::runs::models::RunStatus; -pub(crate) use crate::fault_types::TickFaultTestCase; +pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; pub(crate) struct PriorityClaimsTestCase { diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs index be400b8236..7142cff07b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -1,8 +1,12 @@ +#[path = "store/faults/helpers.rs"] +mod fault_helpers; +#[path = "store/faults/test_types.rs"] +mod fault_types; #[path = "scheduler/helpers.rs"] mod helpers; -#[path = "store/lease_helpers.rs"] +#[path = "store/leases/helpers.rs"] mod lease_helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; @@ -12,3 +16,5 @@ mod store_test_types; mod test_contract; #[path = "scheduler/test_types.rs"] mod test_types; +#[path = "scheduler/ticks/test_ticks.rs"] +mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs similarity index 96% rename from crates/sqlbuild-orchestration/tests/ticks/helpers.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 0ce57192b9..fa51f828f5 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -1,3 +1,5 @@ +//! Trigger and failure fixtures exercise the scheduler's transactional tick boundary. + use std::collections::BTreeMap; use serde_json::json; @@ -20,7 +22,7 @@ use crate::lease_helpers::expire_lease; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; -use crate::test_types::TickFaultTestCase; +use crate::ticks::test_types::TransactionFault; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { @@ -77,7 +79,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi .expect("elected"); let second = second_store(store).await; for point in [ - TickFaultTestCase { + TransactionFault { description: "tick insertion", table: "sqb_ticks", operation: "INSERT", @@ -85,7 +87,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected tick failure')", postgres_body: "RAISE EXCEPTION 'injected tick failure'", }, - TickFaultTestCase { + TransactionFault { description: "requested event", table: "recall_events", operation: "INSERT", @@ -93,7 +95,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected requested failure')", postgres_body: "RAISE EXCEPTION 'injected requested failure'", }, - TickFaultTestCase { + TransactionFault { description: "queued event", table: "recall_events", operation: "INSERT", @@ -101,7 +103,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected queued failure')", postgres_body: "RAISE EXCEPTION 'injected queued failure'", }, - TickFaultTestCase { + TransactionFault { description: "run projection", table: "sqb_runs", operation: "INSERT", @@ -109,7 +111,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected run failure')", postgres_body: "RAISE EXCEPTION 'injected run failure'", }, - TickFaultTestCase { + TransactionFault { description: "node projection", table: "sqb_run_nodes", operation: "INSERT", @@ -117,7 +119,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", postgres_body: "RAISE EXCEPTION 'injected node failure'", }, - TickFaultTestCase { + TransactionFault { description: "cursor commit", table: "sqb_trigger_state", operation: "UPDATE", @@ -125,7 +127,7 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "SELECT RAISE(ABORT,'injected cursor failure')", postgres_body: "RAISE EXCEPTION 'injected cursor failure'", }, - TickFaultTestCase { + TransactionFault { description: "lease expires during append", table: "sqb_runs", operation: "INSERT", diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs similarity index 83% rename from crates/sqlbuild-orchestration/tests/ticks/test_contract.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs index 7d733d45be..af4492a29d 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs @@ -1,7 +1,9 @@ -use crate::helpers::transaction_contract; +//! Crash points must not separate a tick from its queued work and durable cursor. + use crate::store_helpers::run; use crate::store_test_types::Backend; -use crate::test_types::TickTransactionTestCase; +use crate::ticks::helpers::transaction_contract; +use crate::ticks::test_types::TickTransactionTestCase; #[test] fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs similarity index 96% rename from crates/sqlbuild-orchestration/tests/ticks/test_loop.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index 2864da8921..f9341cf027 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -1,3 +1,5 @@ +//! Embedded and standalone schedulers share takeover and shutdown semantics. + use std::time::Duration; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; @@ -8,12 +10,12 @@ use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::snapshot; use crate::lease_helpers::expire_lease; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::TickTransactionTestCase; +use crate::ticks::helpers::snapshot; +use crate::ticks::test_types::TickTransactionTestCase; #[test] fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs new file mode 100644 index 0000000000..3f36d1d4fe --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -0,0 +1,10 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_loop.rs"] +mod test_loop; +#[path = "test_triggers.rs"] +mod test_triggers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs similarity index 96% rename from crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 4429dae9fb..4df7178b12 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -1,3 +1,5 @@ +//! Operator trigger facts remain consistent across publication and retry boundaries. + use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; @@ -12,10 +14,10 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{TickFaultTestCase, TriggerControlsTestCase}; +use crate::ticks::helpers::snapshot; +use crate::ticks::test_types::{TransactionFault, TriggerControlsTestCase}; #[test] fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() @@ -84,7 +86,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::Enable, ..pause.clone() }; - let fault = TickFaultTestCase { + let fault = TransactionFault { description: "operator event and projection", table: "sqb_trigger_state", operation: "UPDATE", diff --git a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs similarity index 78% rename from crates/sqlbuild-orchestration/tests/ticks/test_types.rs rename to crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 9559e8a92f..95953810a5 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -1,4 +1,6 @@ -pub(crate) use crate::fault_types::TickFaultTestCase; +//! Database cases and expected scheduler tick outcomes. + +pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; pub(crate) struct TickTransactionTestCase { diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs index 5dcb1cd22c..58d1347746 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -2,7 +2,7 @@ mod contract; #[path = "snapshots/helpers.rs"] mod helpers; -#[path = "store/replica_helpers.rs"] +#[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] mod store_helpers; diff --git a/crates/sqlbuild-orchestration/tests/store.rs b/crates/sqlbuild-orchestration/tests/store.rs new file mode 100644 index 0000000000..5a79851f21 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store.rs @@ -0,0 +1,16 @@ +//! Recall storage integration contracts for SQLite and PostgreSQL. + +#[path = "store/recall/helpers.rs"] +mod helpers; +#[path = "store/replicas/helpers.rs"] +mod replica_helpers; +#[path = "store/helpers.rs"] +mod store_helpers; +#[path = "store/test_types.rs"] +mod store_test_types; +#[path = "store/recall/test_contract.rs"] +mod test_contract; +#[path = "store/recall/test_types.rs"] +mod test_types; +#[path = "store/recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs similarity index 92% rename from crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs rename to crates/sqlbuild-orchestration/tests/store/faults/helpers.rs index 9c973c0fb1..c7d5c81dd5 100644 --- a/crates/sqlbuild-orchestration/tests/ticks/fault_helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -1,8 +1,10 @@ +//! Database-trigger fault injection at durable publication boundaries. + use sqlbuild_orchestration::store::models::ControlStore; -use crate::test_types::TickFaultTestCase; +use crate::fault_types::TransactionFault; -pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCase) { +pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { unreachable!() @@ -23,7 +25,7 @@ pub(crate) async fn install_fault(store: &ControlStore, point: &TickFaultTestCas .execute(pool).await.expect(point.description); } -pub(crate) async fn remove_fault(store: &ControlStore, point: &TickFaultTestCase) { +pub(crate) async fn remove_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { unreachable!() diff --git a/crates/sqlbuild-orchestration/tests/store/fault_types.rs b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs similarity index 68% rename from crates/sqlbuild-orchestration/tests/store/fault_types.rs rename to crates/sqlbuild-orchestration/tests/store/faults/test_types.rs index eb9e82bd2f..e25282c3cc 100644 --- a/crates/sqlbuild-orchestration/tests/store/fault_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs @@ -1,6 +1,6 @@ -//! Transaction failure points shared by credential, tick and run contracts. +//! Transaction failure cases shared by credential, tick and run contracts. -pub(crate) struct TickFaultTestCase { +pub(crate) struct TransactionFault { pub(crate) description: &'static str, pub(crate) table: &'static str, pub(crate) operation: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs b/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs deleted file mode 100644 index 8e17cfce86..0000000000 --- a/crates/sqlbuild-orchestration/tests/store/lease_helpers.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Expire the persisted lease without introducing a fake control-store clock. - -use sqlbuild_orchestration::store::models::ControlStore; - -pub(crate) async fn expire_lease(store: &ControlStore) { - match store { - ControlStore::Sqlite(path) => { - rusqlite::Connection::open(path) - .expect("open lease expiry connection") - .execute_batch( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .expect("expire SQLite scheduler lease"); - } - ControlStore::Postgres(pool) => { - sqlx_core::query::query( - "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", - ) - .execute(pool) - .await - .expect("expire PostgreSQL scheduler lease"); - } - } -} diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs new file mode 100644 index 0000000000..b813d957d3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -0,0 +1,24 @@ +//! Expire the persisted lease without introducing a fake control-store clock. + +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open lease expiry connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite scheduler lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); +} diff --git a/crates/sqlbuild-orchestration/tests/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs similarity index 98% rename from crates/sqlbuild-orchestration/tests/recall/helpers.rs rename to crates/sqlbuild-orchestration/tests/store/recall/helpers.rs index ce7de75ac3..2fb0605a1d 100644 --- a/crates/sqlbuild-orchestration/tests/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs @@ -1,3 +1,5 @@ +//! Recall publication fixtures exercise the control store's durable append boundary. + use serde_json::json; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::publish_recall::publish_recall; diff --git a/crates/sqlbuild-orchestration/tests/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs similarity index 93% rename from crates/sqlbuild-orchestration/tests/recall/test_contract.rs rename to crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs index d8d8bf5629..d0ccf95c46 100644 --- a/crates/sqlbuild-orchestration/tests/recall/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -1,3 +1,5 @@ +//! Idempotency and commit-order paging through the public store API. + use crate::helpers::recall_contract; use crate::store_helpers::run; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs similarity index 84% rename from crates/sqlbuild-orchestration/tests/recall/test_types.rs rename to crates/sqlbuild-orchestration/tests/store/recall/test_types.rs index 8ecb5f4f63..481cac0601 100644 --- a/crates/sqlbuild-orchestration/tests/recall/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -1,3 +1,5 @@ +//! Expected Recall storage outcomes for both supported databases. + use crate::store_test_types::Backend; pub(crate) struct RecallTestCase { diff --git a/crates/sqlbuild-orchestration/tests/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs similarity index 99% rename from crates/sqlbuild-orchestration/tests/recall/test_views.rs rename to crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index 0dd0f229a5..75a8b9971d 100644 --- a/crates/sqlbuild-orchestration/tests/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -1,3 +1,5 @@ +//! Durable Recall views preserve the published execution records. + use serde_json::json; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::publish_recall::publish_recall; diff --git a/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs b/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs deleted file mode 100644 index 853ba75df8..0000000000 --- a/crates/sqlbuild-orchestration/tests/store/replica_helpers.rs +++ /dev/null @@ -1,15 +0,0 @@ -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; - -pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { - match store { - ControlStore::Sqlite(path) => open_store(path.to_str().expect("UTF-8 test path")) - .await - .expect("open independent SQLite handle"), - ControlStore::Postgres(pool) => ControlStore::Postgres( - sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) - .await - .expect("open independent PostgreSQL pool"), - ), - } -} diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs new file mode 100644 index 0000000000..22fe1b374b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -0,0 +1,20 @@ +//! Independent database handles for multi-replica integration contracts. + +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"); + }; + ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/ticks.rs b/crates/sqlbuild-orchestration/tests/ticks.rs deleted file mode 100644 index abf2590859..0000000000 --- a/crates/sqlbuild-orchestration/tests/ticks.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[path = "ticks/fault_helpers.rs"] -mod fault_helpers; -#[path = "store/fault_types.rs"] -mod fault_types; -#[path = "ticks/helpers.rs"] -mod helpers; -#[path = "store/lease_helpers.rs"] -mod lease_helpers; -#[path = "store/replica_helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "ticks/test_contract.rs"] -mod test_contract; -#[path = "ticks/test_loop.rs"] -mod test_loop; -#[path = "ticks/test_triggers.rs"] -mod test_triggers; -#[path = "ticks/test_types.rs"] -mod test_types; From 975eed10462a61e6d8364cc7a38eceb0ca191f55 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:14:37 +0000 Subject: [PATCH 010/138] feat: monitor silent workers during scheduler cycles Observe heartbeat loss in fenced, bounded transactions while retaining physical write claims. Recover through existing worker heartbeats and continue full batches without an idle delay. --- .../src/runs/_helpers/mod.rs | 1 + .../src/runs/_helpers/publish_change.rs | 54 +++ .../src/runs/constants.rs | 3 + .../src/runs/main/change_run.rs | 42 +-- .../src/runs/main/mod.rs | 1 + .../src/runs/main/monitor_runs.rs | 55 +++ .../scheduler/_helpers/drain_due_triggers.rs | 37 ++ .../src/scheduler/_helpers/mod.rs | 1 + .../src/scheduler/main/scheduler_cycle.rs | 32 +- .../src/scheduler/models.rs | 1 + crates/sqlbuild-orchestration/tests/runs.rs | 4 + .../tests/runs/monitor/helpers.rs | 70 ++++ .../tests/runs/test_monitor.rs | 343 ++++++++++++++++++ .../tests/runs/test_types.rs | 13 + 14 files changed, 597 insertions(+), 60 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_monitor.rs diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index 5994d61750..75a1590516 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod claims; pub(crate) mod events; pub(crate) mod nodes; pub(crate) mod projection; +pub(crate) mod publish_change; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs new file mode 100644 index 0000000000..18ba3f5835 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs @@ -0,0 +1,54 @@ +//! Publish a lifecycle fact, its projection and terminal cleanup in the caller's transaction. + +use serde_json::json; + +use crate::runs::_helpers::events::append_run_event; +use crate::runs::_helpers::projection::project_change; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_change( + transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let (transaction, mut cursor) = append_run_event( + transaction, + (request.run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + let mut transaction = project_change(transaction, request, state).await?; + if matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none() + { + let terminal = RunChangeRequest { + event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), + run_id: request.run_id, + claim_id: None, + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "cancelled before launch".into(), + exit_code: None, + }, + }; + (transaction, cursor) = append_run_event( + transaction, + (terminal.run_id, terminal.event_id), + terminal.change.event_type(), + json!({"change": terminal}), + ) + .await?; + transaction = project_change(transaction, &terminal, state).await?; + } + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index d92a2e052a..801d9c9edb 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -2,3 +2,6 @@ pub(crate) const NODE_PENDING: &str = "pending"; pub(crate) const NODE_RUNNING: &str = "running"; + +pub(crate) const HEARTBEAT_TIMEOUT_SECONDS: i64 = 45; +pub(crate) const MONITOR_BATCH_SIZE: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 3bef1033c8..02fe0308c1 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -3,9 +3,8 @@ use serde_json::json; use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::_helpers::projection::project_change; -use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -50,42 +49,7 @@ pub async fn change_run( .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, )?; validate_change(&state, request)?; - let (transaction, mut cursor) = append_run_event( - transaction, - (request.run_id, request.event_id), - request.change.event_type(), - body, - ) - .await?; - let mut transaction = project_change(transaction, request, &state).await?; - if matches!(request.change, RunChange::CancelRequested { .. }) - && state.worker_instance.is_none() - { - let terminal = RunChangeRequest { - event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), - run_id: request.run_id, - claim_id: None, - change: RunChange::Finished { - outcome: RunOutcome::Cancelled, - reason: "cancelled before launch".into(), - exit_code: None, - }, - }; - (transaction, cursor) = append_run_event( - transaction, - (terminal.run_id, terminal.event_id), - terminal.change.event_type(), - json!({"change": terminal}), - ) - .await?; - transaction = project_change(transaction, &terminal, &state).await?; - } - let (mut transaction, _) = query( - transaction, - "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", - vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], - ) - .await?; + let mut transaction = publish_change(transaction, request, &state).await?; if let RunAuthority::Scheduler(lease) = authority { transaction = check_lease(transaction, lease).await?; } diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index c19b71681a..ffceba28a4 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -3,6 +3,7 @@ pub mod claim_next_run; pub mod heartbeat_run; pub mod list_run_nodes; pub mod list_runs; +pub mod monitor_runs; pub(crate) mod queue_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs new file mode 100644 index 0000000000..b23d4eb791 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs @@ -0,0 +1,55 @@ +//! Observe heartbeat loss under the scheduler fence without releasing a live worker's claims. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::constants::{HEARTBEAT_TIMEOUT_SECONDS, MONITOR_BATCH_SIZE}; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Serialized writers make heartbeat inspection and the resulting fact one atomic observation. +pub async fn monitor_runs( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let cutoff = transaction.now - TimeDelta::seconds(HEARTBEAT_TIMEOUT_SECONDS); + let rows; + (transaction, rows) = query( + transaction, + &format!( + "SELECT r.run_id, r.status, r.cancel_requested_at, r.node_counts_json, \ + r.last_event_storage_id, l.worker_instance FROM sqb_runs r \ + JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.status='running' AND l.last_heartbeat_at<$1 \ + ORDER BY l.last_heartbeat_at, r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![Parameter::Time(cutoff)], + ) + .await?; + let observed = rows.len(); + for row in rows { + let run_id: Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let previous: i64 = serde_json::from_value(row["last_event_storage_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let state: ChangingRun = decode_row(row)?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&run_id, format!("unresponsive/{previous}").as_bytes()), + run_id, + claim_id: state.worker_instance, + change: RunChange::Unresponsive, + }; + transaction = publish_change(transaction, &request, &state).await?; + } + commit(check_lease(transaction, lease).await?).await?; + Ok(observed) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs new file mode 100644 index 0000000000..ab6a639022 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -0,0 +1,37 @@ +//! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. + +use uuid::Uuid; + +use crate::scheduler::main::evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_due_triggers( + store: &ControlStore, + mut lease: SchedulerLease, + mut cycle: SchedulerCycle, +) -> Result<(SchedulerLease, SchedulerCycle), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + for row in rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { + cycle.evaluated += 1; + cycle.ticks += plan.actions.len(); + } + lease = renew_lease(store, &lease).await?; + } + Ok((lease, cycle)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs index 85bb8cc89b..5102baaac9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod calendar; +pub(crate) mod drain_due_triggers; pub(crate) mod expansion; pub(crate) mod persistence; pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs index 011d28e303..8db0fea9b5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs @@ -3,18 +3,19 @@ use std::time::Duration; use chrono::{DateTime, Utc}; -use uuid::Uuid; +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::scheduler::_helpers::drain_due_triggers::drain_due_triggers; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; use crate::scheduler::main::acquire_lease::acquire_lease; -use crate::scheduler::main::evaluate_trigger::evaluate_trigger; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn scheduler_cycle( store: &ControlStore, @@ -29,31 +30,20 @@ pub async fn scheduler_cycle( Err(ControlError::Fenced) => None, Err(error) => return Err(error), }; - let mut cycle = SchedulerCycle { + let cycle = SchedulerCycle { leader: state.lease.is_some(), evaluated: 0, ticks: 0, + unresponsive: 0, wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), }; - let Some(mut lease) = state.lease.clone() else { + let Some(lease) = state.lease.clone() else { return Ok((state, cycle)); }; - let transaction = begin(store, TransactionMode::Read).await?; - let now = transaction.now; - let (transaction, rows) = query(transaction, - "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ - JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ - ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; - commit(transaction).await?; - for row in rows { - let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { - cycle.evaluated += 1; - cycle.ticks += plan.actions.len(); - } - lease = renew_lease(store, &lease).await?; + let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?; + cycle.unresponsive = monitor_runs(store, &lease).await?; + if cycle.unresponsive == MONITOR_BATCH_SIZE { + cycle.wait_for = Duration::ZERO; } state.lease = Some(lease); let transaction = begin(store, TransactionMode::Read).await?; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index a3a0c2ccad..019083278d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -36,6 +36,7 @@ pub struct SchedulerCycle { pub leader: bool, pub evaluated: usize, pub ticks: usize, + pub unresponsive: usize, pub wait_for: std::time::Duration, } diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 5b2f42488c..12d6c66d52 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -6,6 +6,8 @@ mod fault_types; mod helpers; #[path = "store/leases/helpers.rs"] mod lease_helpers; +#[path = "runs/monitor/helpers.rs"] +mod monitor_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] @@ -22,5 +24,7 @@ mod test_coalesce; mod test_faults; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; +#[path = "runs/test_monitor.rs"] +mod test_monitor; #[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs new file mode 100644 index 0000000000..7f90e2f877 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -0,0 +1,70 @@ +//! Advance persisted liveness, not the scheduler clock, to exercise real monitoring transactions. + +use std::collections::BTreeMap; + +use chrono::{TimeDelta, Utc}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest, +}; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::helpers::queue; + +pub(crate) async fn start_run( + store: &ControlStore, + lease: &SchedulerLease, + input: &RunRequest, +) -> RunClaim { + queue(store, input).await; + let claim = claim_next_run(store, lease, &BTreeMap::new()) + .await + .expect("claim queued run") + .expect("run available"); + assert_eq!(claim.request.run_id, input.run_id); + change_run( + store, + RunAuthority::Worker { + run_id: input.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start claimed worker"); + claim +} + +pub(crate) async fn age_heartbeat(store: &ControlStore, run_id: Uuid, seconds: i64) { + let timestamp = Utc::now() - TimeDelta::seconds(seconds); + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open liveness fixture") + .execute( + "UPDATE sqb_run_liveness SET last_heartbeat_at=?1 WHERE run_id=?2", + rusqlite::params![ + timestamp.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + run_id.to_string() + ], + ) + .expect("age SQLite heartbeat"); + return; + }; + sqlx_core::query::query("UPDATE sqb_run_liveness SET last_heartbeat_at=$1 WHERE run_id=$2") + .bind(timestamp) + .bind(run_id) + .execute(pool) + .await + .expect("age PostgreSQL heartbeat"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs new file mode 100644 index 0000000000..c4847d7316 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -0,0 +1,343 @@ +//! Real scheduler cycles detect silent workers without surrendering their physical write claims. + +use std::collections::BTreeMap; +use std::time::Duration; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::main::monitor_runs::monitor_runs; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::{queue, request, snapshot}; +use crate::lease_helpers::expire_lease; +use crate::monitor_helpers::{age_heartbeat, start_run}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault}; + +#[test] +fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_facts_retain_claims_and_allow_recovery() + { + let test_cases = [ + RunMonitorTestCase { + description: "SQLite observes only running workers with expired heartbeats", + backend: Backend::Sqlite, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + RunMonitorTestCase { + description: "PostgreSQL observes only running workers with expired heartbeats", + backend: Backend::Postgres, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let holder = Uuid::now_v7(); + let lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + let silent = start_run(store, &lease, &request(scope, 1, 0, &["orders"])).await; + start_run(store, &lease, &request(scope, 2, 0, &["customers"])).await; + age_heartbeat(store, Uuid::from_u128(2), 40).await; + queue(store, &request(scope, 3, 0, &["inventory"])).await; + let launching = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("launching"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: launching.request.run_id, + claim_id: Some(launching.claim_id), + change: RunChange::Launched { + container_id: Some("orders-worker".into()), + }, + }, + ) + .await + .expect("launcher acknowledges"); + let failed = start_run(store, &lease, &request(scope, 4, 0, &["products"])).await; + change_run( + store, + RunAuthority::Worker { + run_id: failed.request.run_id, + claim_id: failed.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: failed.request.run_id, + claim_id: Some(failed.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "worker failed".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminal run"); + queue(store, &request(scope, 5, 0, &["orders"])).await; + age_heartbeat(store, silent.request.run_id, 50).await; + age_heartbeat(store, launching.request.run_id, 50).await; + let before = read_recall_events(store, 0, 100) + .await + .expect("before monitoring"); + let projection = list_runs(store, (location, "development"), 100) + .await + .expect("before monitoring"); + let fault = TransactionFault { + description: "monitor projection failure rolls back the observation", + table: "sqb_runs", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected monitor failure')", + postgres_body: "RAISE EXCEPTION 'injected monitor failure'", + }; + install_fault(store, &fault).await; + assert!( + monitor_runs(store, &lease).await.is_err(), + "{}", + test_case.description + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("projection rollback"), + projection + ); + expire_lease(store).await; + let (state, observed) = scheduler_cycle(store, SchedulerState::new(holder)) + .await + .expect("monitoring cycle"); + assert_eq!(observed.unresponsive, 1, "{}", test_case.description); + let statuses: BTreeMap<_, _> = list_runs(store, (location, "development"), 100) + .await + .expect("monitored statuses") + .into_iter() + .map(|record| (record.run_id, record.status)) + .collect(); + assert_eq!( + std::array::from_fn(|index| statuses[&Uuid::from_u128(index as u128 + 1)]), + test_case.expected_statuses, + "{}", + test_case.description + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("observation"); + assert_eq!(recorded.len(), before.len() + 1); + assert_eq!( + recorded.last().expect("fact")["event_type"], + "run_unresponsive" + ); + let (state, repeated) = scheduler_cycle(store, state) + .await + .expect("repeated monitor"); + assert_eq!(repeated.unresponsive, 0); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no repeated facts"), + recorded + ); + let (_, follower) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("follower"); + assert!(!follower.leader); + assert_eq!(follower.unresponsive, 0); + assert!( + !heartbeat_run(store, (silent.request.run_id, silent.claim_id)) + .await + .expect("worker recovers") + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("recovered projection") + .into_iter() + .find(|record| record.run_id == silent.request.run_id) + .expect("recovered run") + .status, + RunStatus::Running + ); + age_heartbeat(store, silent.request.run_id, 50).await; + let (_, second_loss) = scheduler_cycle(store, state) + .await + .expect("new heartbeat loss"); + assert_eq!(second_loss.unresponsive, 1); + let events = read_recall_events(store, 0, 100) + .await + .expect("successive losses"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_recovered") + .count(), + 1 + ); + expire_lease(store).await; + assert!(matches!( + monitor_runs(store, &lease).await, + Err(ControlError::Fenced) + )); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fenced monitor"), + events + ); + let successor = acquire_lease(store, Uuid::now_v7()) + .await + .expect("takeover") + .expect("successor"); + assert!( + claim_next_run(store, &successor, &BTreeMap::new()) + .await + .expect("write claims remain held after heartbeat loss and takeover") + .is_none() + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_silent_worker_backlog_when_monitoring_then_bounded_transactions_continue_immediately_until_drained() + { + let test_cases = [ + MonitorBacklogTestCase { + description: "SQLite drains more than one monitor page", + backend: Backend::Sqlite, + run_count: 105, + expected_batches: [100, 5, 0], + }, + MonitorBacklogTestCase { + description: "PostgreSQL drains more than one monitor page", + backend: Backend::Postgres, + run_count: 105, + expected_batches: [100, 5, 0], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let holder = Uuid::now_v7(); + let mut lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + for index in 0..test_case.run_count { + let mut input = request( + (location, published.snapshot_id), + index as u128 + 1, + 0, + &["orders"], + ); + input.write_set.clear(); + start_run(store, &lease, &input).await; + age_heartbeat(store, input.run_id, 50).await; + lease = renew_lease(store, &lease) + .await + .expect("renew during setup"); + } + expire_lease(store).await; + let mut state = SchedulerState::new(holder); + let mut batches = Vec::new(); + for expected in test_case.expected_batches { + let cycle; + (state, cycle) = scheduler_cycle(store, state).await.expect("monitor batch"); + batches.push(cycle.unresponsive); + assert_eq!( + cycle.wait_for, + if expected == 100 { + Duration::ZERO + } else { + Duration::from_secs(5) + } + ); + } + assert_eq!( + batches, test_case.expected_batches, + "{}", + test_case.description + ); + let events = read_recall_events(store, 0, 1000) + .await + .expect("monitor facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + test_case.run_count + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 81471a4f67..f0112b2903 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -42,3 +42,16 @@ pub(crate) struct RunLifecycleTestCase { pub(crate) backend: Backend, pub(crate) expected_status: RunStatus, } + +pub(crate) struct RunMonitorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_statuses: [RunStatus; 5], +} + +pub(crate) struct MonitorBacklogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) run_count: usize, + pub(crate) expected_batches: [usize; 3], +} From 6981f103eeadef23e22c3f57ce4c618bfc014aff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:35:59 +0000 Subject: [PATCH 011/138] feat: keep worker heartbeats independent of execution Own worker heartbeats on a dedicated native runtime and connection pool. Propagate cancellation and revoked credentials without cancelling work on transient API failures; stop and join with the execution owner. --- crates/sqlbuild-cli/tests/native_commands.rs | 2 + .../tests/native_commands/worker/helpers.rs | 180 ++++++++++++++++++ .../native_commands/worker/test_heartbeat.rs | 63 ++++++ .../native_commands/worker/test_types.rs | 14 ++ .../native_commands/worker/test_worker.rs | 6 + .../src/worker/_helpers/heartbeat.rs | 90 +++++++++ .../src/worker/_helpers/mod.rs | 2 + .../src/worker/_helpers/response.rs | 15 ++ .../src/worker/_helpers/session.rs | 22 +++ .../src/worker/errors.rs | 2 + .../src/worker/main/connect_worker.rs | 15 +- .../src/worker/main/start_worker.rs | 18 +- .../src/worker/models.rs | 15 ++ .../tests/api/test_worker.rs | 12 +- 14 files changed, 431 insertions(+), 25 deletions(-) create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/session.rs diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index 23a9b5ba97..e93113875b 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -4,3 +4,5 @@ mod helpers; mod test_control; #[path = "native_commands/test_types.rs"] mod test_types; +#[path = "native_commands/worker/test_worker.rs"] +mod worker; diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs new file mode 100644 index 0000000000..2460bfd6ab --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -0,0 +1,180 @@ +//! Privileged test operators stay separate from the API-only worker session. + +use std::collections::BTreeMap; +use std::time::{Duration, SystemTime}; + +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunKind, RunOrigin, RunRequest, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::models::Overlap; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::main::create_location::create_location; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::WorkerSession; +use uuid::Uuid; + +use crate::test_types::TestControl; +use crate::worker::test_types::WorkerControl; + +pub(crate) async fn start_claimed_worker( + control: &TestControl, + url: &str, +) -> (WorkerSession, WorkerControl) { + let path = control.directory.path().join(".sqlbuild/control.sqlite3"); + let store = open_store(path.to_str().expect("test database path")) + .await + .expect("native control store"); + let location_id = create_location(&store, "orders", "test-admin") + .await + .expect("location"); + let baseline = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::new(), + }; + let publication = publish_snapshot(&store, &baseline).await.expect("snapshot"); + let run_id = Uuid::now_v7(); + let request = RunRequest { + run_id, + location_id, + target: "development".into(), + snapshot_id: publication.snapshot_id, + image_id: None, + kind: RunKind::Adhoc, + origin: RunOrigin::Manual, + job_key: None, + parent_run_id: None, + root_run_id: run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: "test-admin".into(), + priority: 0, + selection: ["orders".into()].into(), + write_set: ["orders".into()].into(), + settings: BTreeMap::new(), + concurrency_groups: Default::default(), + overlap: Overlap::Allow, + }; + request_run(&store, &request).await.expect("queue run"); + let lease = acquire_lease(&store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("leader"); + let claim = claim_next_run(&store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("queued work"); + let identity = (run_id, claim.claim_id); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + &store, + &context, + CredentialScope::Run { + run_id, + claim_id: identity.1, + }, + (SystemTime::now() + Duration::from_secs(3600)).into(), + ) + .await + .expect("run-scoped token"); + change_run( + &store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("launched worker"); + let client = connect_worker(url, &token.bearer, identity).expect("worker connection"); + let session = start_worker(&client, &baseline) + .await + .expect("verified worker"); + ( + session, + WorkerControl { + store, + context, + token_id: token.identity.token_id, + identity, + }, + ) +} + +pub(crate) fn cancel_worker(worker: WorkerControl, _control: &mut TestControl) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("independent operator runtime"); + runtime + .block_on(change_run( + &worker.store, + RunAuthority::Operator("test-admin"), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: worker.identity.0, + claim_id: Some(worker.identity.1), + change: RunChange::CancelRequested { + actor: "test-admin".into(), + }, + }, + )) + .expect("request cancellation while worker runtime is blocked"); +} + +pub(crate) fn revoke_worker(worker: WorkerControl, _control: &mut TestControl) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("independent operator runtime"); + assert!( + runtime + .block_on(revoke_token( + &worker.store, + &worker.context, + worker.token_id + )) + .expect("revoke worker credential") + ); +} + +pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestControl) { + control.child.kill().expect("stop API process"); + control.child.wait().expect("reap API process"); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs new file mode 100644 index 0000000000..011e0e5490 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs @@ -0,0 +1,63 @@ +//! The standalone server remains runnable while the worker's runtime is deliberately blocked. + +use std::time::{Duration, Instant}; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + cancel_worker, disconnect_worker, revoke_worker, start_claimed_worker, +}; +use crate::worker::test_types::WorkerHeartbeatTestCase; + +#[test] +fn given_blocking_worker_work_when_control_changes_then_heartbeats_remain_independent() { + let test_cases = [ + WorkerHeartbeatTestCase { + description: "cancellation reaches a blocked worker runtime", + interruption: cancel_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "credential revocation reaches a blocked worker runtime", + interruption: revoke_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "temporary API unavailability does not cancel warehouse work", + interruption: disconnect_worker, + expected_cancelled: false, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("worker runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (session, worker) = start_claimed_worker(&control, &url).await; + let operator = std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(2)); + (test_case.interruption)(worker, &mut control); + control + }); + // No Tokio IO can run on this caller until after the assertion. A heartbeat spawned + // on this runtime, or sharing its connection pool, cannot deliver cancellation. + std::thread::sleep(Duration::from_secs(12)); + assert_eq!( + session.cancel_requested(), + test_case.expected_cancelled, + "{}", + test_case.description + ); + let stopping = Instant::now(); + drop(session); + assert!( + stopping.elapsed() < Duration::from_secs(2), + "{}", + test_case.description + ); + drop(operator.join().expect("operator completed")); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs new file mode 100644 index 0000000000..73fb9bf997 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -0,0 +1,14 @@ +use crate::test_types::TestControl; + +pub(crate) struct WorkerHeartbeatTestCase { + pub(crate) description: &'static str, + pub(crate) interruption: fn(WorkerControl, &mut TestControl), + pub(crate) expected_cancelled: bool, +} + +pub(crate) struct WorkerControl { + pub(crate) store: sqlbuild_orchestration::store::models::ControlStore, + pub(crate) context: sqlbuild_orchestration::credentials::models::AuditContext, + pub(crate) token_id: uuid::Uuid, + pub(crate) identity: (uuid::Uuid, uuid::Uuid), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs new file mode 100644 index 0000000000..13b54de371 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs @@ -0,0 +1,6 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_heartbeat.rs"] +mod test_heartbeat; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs new file mode 100644 index 0000000000..a8afb1fce8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs @@ -0,0 +1,90 @@ +//! A dedicated native runtime keeps heartbeat IO independent of warehouse work and Python's GIL. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use crate::runs::models::RunExecution; +use crate::worker::_helpers::response::http_client; +use crate::worker::errors::WorkerError; +use crate::worker::models::{WorkerClient, WorkerSession}; + +async fn heartbeat_loop( + client: WorkerClient, + cancelled: Arc, + mut stop: tokio::sync::watch::Receiver, +) { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = stop.changed() => return, + _ = interval.tick() => {} + } + let result = tokio::select! { + biased; + _ = stop.changed() => return, + result = client.heartbeat() => result, + }; + match result { + Ok(true) => cancelled.store(true, Ordering::Release), + Err(WorkerError::Rejected(status)) + if matches!(status.as_u16(), 401 | 403 | 404 | 410) => + { + cancelled.store(true, Ordering::Release); + return; + } + Ok(false) | Err(_) => {} + } + } +} + +pub(crate) async fn start_session( + client: WorkerClient, + execution: RunExecution, +) -> Result { + let cancelled = Arc::new(AtomicBool::new(false)); + let signal = Arc::clone(&cancelled); + let (stop, stopping) = tokio::sync::watch::channel(false); + let (ready, started) = tokio::sync::oneshot::channel(); + let thread = std::thread::Builder::new() + .name("sqb-worker-heartbeat".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready.send(Err(WorkerError::from(error))); + return; + } + }; + runtime.block_on(async move { + let heartbeat = match http_client(client.authorization.clone()) { + Ok(http) => WorkerClient { + client: http, + ..client + }, + Err(error) => { + let _ = ready.send(Err(error)); + return; + } + }; + let _ = ready.send(Ok(())); + heartbeat_loop(heartbeat, signal, stopping).await; + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + })?; + let session = WorkerSession { + execution, + cancel_requested: cancelled, + stop, + thread: Some(thread), + }; + started + .await + .map_err(|_| WorkerError::Configuration("heartbeat thread stopped during startup"))??; + Ok(session) +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs index 57d94ff592..1bbe75bead 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod client; +pub(crate) mod heartbeat; pub(crate) mod response; +pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs index e4da90cca5..0790ae0288 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs @@ -1,9 +1,24 @@ //! Do not echo response bodies: server diagnostics can contain user configuration. +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; use serde::de::DeserializeOwned; use crate::worker::errors::WorkerError; +/// Each runtime owns its connection pool; sharing pools could tie heartbeat IO to build work. +pub(crate) fn http_client(authorization: HeaderValue) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + Ok(reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?) +} + pub(crate) async fn response( request: reqwest::RequestBuilder, ) -> Result { diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs new file mode 100644 index 0000000000..e7bdd3ad08 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs @@ -0,0 +1,22 @@ +//! The execution owner drains its heartbeat before worker runtime teardown. + +use std::sync::atomic::Ordering; + +use crate::worker::models::WorkerSession; + +impl WorkerSession { + /// Cancellation and credential revocation are monotonic for one pinned worker invocation. + #[must_use] + pub fn cancel_requested(&self) -> bool { + self.cancel_requested.load(Ordering::Acquire) + } +} + +impl Drop for WorkerSession { + fn drop(&mut self) { + let _ = self.stop.send(true); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs index 30a6a18f8e..e5e62c936a 100644 --- a/crates/sqlbuild-orchestration/src/worker/errors.rs +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -8,6 +8,8 @@ pub enum WorkerError { Transport(#[from] reqwest::Error), #[error("worker API rejected the request with HTTP {0}")] Rejected(reqwest::StatusCode), + #[error("worker heartbeat could not start: {0}")] + Heartbeat(#[from] std::io::Error), #[error(transparent)] Verification(#[from] crate::store::errors::ControlError), #[error("run cancellation was requested before execution")] diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs index f1c543d1dc..cf90ad449f 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -1,10 +1,9 @@ //! Bound worker requests have deadlines and must never follow credential-bearing redirects. -use std::time::Duration; - -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use reqwest::header::HeaderValue; use uuid::Uuid; +use crate::worker::_helpers::response::http_client; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; @@ -37,16 +36,10 @@ pub fn connect_worker( let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) .map_err(|_| WorkerError::Configuration("run bearer is not a valid HTTP credential"))?; authorization.set_sensitive(true); - let mut headers = HeaderMap::new(); - headers.insert(AUTHORIZATION, authorization); - let client = reqwest::Client::builder() - .default_headers(headers) - .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(30)) - .connect_timeout(Duration::from_secs(10)) - .build()?; + let client = http_client(authorization.clone())?; Ok(WorkerClient { client, + authorization, run_url, run_id, claim_id, diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs index c5710f3e31..72569448be 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -2,27 +2,25 @@ use uuid::Uuid; -use crate::runs::models::{RunChange, RunExecution}; +use crate::runs::models::RunChange; use crate::snapshots::main::verify_execution::verify_execution; use crate::snapshots::models::SnapshotInput; +use crate::worker::_helpers::heartbeat::start_session; use crate::worker::errors::WorkerError; -use crate::worker::models::WorkerClient; +use crate::worker::models::{WorkerClient, WorkerSession}; /// Verify the baseline compile before planning, cloning, applying run settings or executing. pub async fn start_worker( client: &WorkerClient, baseline: &SnapshotInput, -) -> Result { +) -> Result { let expected = client.execution().await?; verify_execution(baseline, &expected)?; if client.heartbeat().await? { return Err(WorkerError::Cancelled); } - client - .report( - Uuid::new_v5(&expected.claim.claim_id, b"worker-started"), - RunChange::Started, - ) - .await?; - Ok(expected) + let event_id = Uuid::new_v5(&expected.claim.claim_id, b"worker-started"); + let session = start_session(client.clone(), expected).await?; + client.report(event_id, RunChange::Started).await?; + Ok(session) } diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs index 813b4b61e3..02e05c2e89 100644 --- a/crates/sqlbuild-orchestration/src/worker/models.rs +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -1,11 +1,26 @@ //! A worker owns only an API credential, never a control-store connection. +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + use uuid::Uuid; +use crate::runs::models::RunExecution; + #[derive(Clone, Debug)] pub struct WorkerClient { pub(crate) client: reqwest::Client, + pub(crate) authorization: reqwest::header::HeaderValue, pub(crate) run_url: reqwest::Url, pub(crate) run_id: Uuid, pub(crate) claim_id: Uuid, } + +/// Keep this owner alive through execution and terminal reporting, then stop and join its heartbeat. +#[derive(Debug)] +pub struct WorkerSession { + pub execution: RunExecution, + pub(crate) cancel_requested: Arc, + pub(crate) stop: tokio::sync::watch::Sender, + pub(crate) thread: Option>, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index dd61d4724c..5cb27bd833 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -137,13 +137,16 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a ) .await .expect("record process launch"); - let execution = start_worker(&worker, &baseline) + let session = start_worker(&worker, &baseline) .await .expect("verified start"); - assert_eq!(execution.claim, claim, "{}", test_case.description); + assert_eq!(session.execution.claim, claim, "{}", test_case.description); assert_eq!( - start_worker(&worker, &baseline).await.expect("retry start"), - execution, + start_worker(&worker, &baseline) + .await + .expect("retry start") + .execution, + session.execution, "{}", test_case.description ); @@ -300,6 +303,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a "{}", test_case.description ); + drop(session); stop_api(api).await; close_test_store(fixture).await; }); From 70e56d36bc5535ff6cae5f0e354754fc49130b3f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:41:31 +0000 Subject: [PATCH 012/138] refactor: share authenticated control-plane transport Move bounded client construction, credential redaction, URL validation and JSON response handling to the API owner so worker and location flows can share them. Keep independent heartbeat pools and worker-facing error categories. --- .../src/api/_helpers/client.rs | 38 +++++++++++++++++++ .../src/api/_helpers/mod.rs | 1 + .../sqlbuild-orchestration/src/api/errors.rs | 10 +++++ .../src/api/main/client_response.rs | 15 ++++++++ .../src/api/main/connect_client.rs | 31 +++++++++++++++ .../src/api/main/mod.rs | 2 + .../sqlbuild-orchestration/src/api/models.rs | 7 ++++ .../src/worker/_helpers/client.rs | 23 +++++++---- .../src/worker/_helpers/heartbeat.rs | 9 ++--- .../src/worker/_helpers/mod.rs | 1 - .../src/worker/_helpers/response.rs | 30 --------------- .../src/worker/errors.rs | 10 +++++ .../src/worker/main/connect_worker.rs | 32 ++-------------- .../src/worker/models.rs | 4 +- 14 files changed, 140 insertions(+), 73 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/client.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/client_response.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/connect_client.rs delete mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/response.rs diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/client.rs b/crates/sqlbuild-orchestration/src/api/_helpers/client.rs new file mode 100644 index 0000000000..98532b9054 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/client.rs @@ -0,0 +1,38 @@ +//! Each client runtime owns its pool; credentials and server response bodies stay out of diagnostics. + +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; + +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn http_client(authorization: HeaderValue) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + Ok(reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?) +} + +impl ControlClient { + pub(crate) fn independent(&self) -> Result { + Ok(Self { + client: http_client(self.authorization.clone())?, + authorization: self.authorization.clone(), + base_url: self.base_url.clone(), + }) + } + + pub(crate) fn endpoint(&self, segments: &[&str]) -> Result { + let mut url = self.base_url.clone(); + url.path_segments_mut() + .map_err(|_| ClientError::Configuration("server URL cannot contain an API path"))? + .pop_if_empty() + .extend(segments.iter().copied()); + Ok(url) + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 41d8c461c5..4f00fe75c5 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod authentication; +pub(crate) mod client; pub(crate) mod runs; pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs index 011ed4974a..ea9da3591d 100644 --- a/crates/sqlbuild-orchestration/src/api/errors.rs +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -7,6 +7,16 @@ use axum::response::{IntoResponse, Response}; use crate::api::models::ApiErrorBody; use crate::store::errors::ControlError; +#[derive(Debug, thiserror::Error)] +pub enum ClientError { + #[error("invalid API connection: {0}")] + Configuration(&'static str), + #[error("API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), +} + #[derive(Debug)] pub struct ApiError { pub(crate) status: StatusCode, diff --git a/crates/sqlbuild-orchestration/src/api/main/client_response.rs b/crates/sqlbuild-orchestration/src/api/main/client_response.rs new file mode 100644 index 0000000000..2eb643424f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/client_response.rs @@ -0,0 +1,15 @@ +//! Share bounded JSON transport without exposing server bodies or credential-bearing URLs. + +use serde::de::DeserializeOwned; + +use crate::api::errors::ClientError; + +pub(crate) async fn client_response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await.map_err(reqwest::Error::without_url)?; + if !response.status().is_success() { + return Err(ClientError::Rejected(response.status())); + } + Ok(response.json().await.map_err(reqwest::Error::without_url)?) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs new file mode 100644 index 0000000000..c0b5258123 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -0,0 +1,31 @@ +//! Validate the API origin before attaching a sensitive bearer credential. + +use reqwest::header::HeaderValue; + +use crate::api::_helpers::client::http_client; +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result { + let base_url = reqwest::Url::parse(server_url) + .map_err(|_| ClientError::Configuration("server URL must be an absolute HTTP URL"))?; + if !matches!(base_url.scheme(), "http" | "https") + || base_url.host_str().is_none() + || !base_url.username().is_empty() + || base_url.password().is_some() + || base_url.query().is_some() + || base_url.fragment().is_some() + { + return Err(ClientError::Configuration( + "server URL must use HTTP or HTTPS without credentials, query or fragment", + )); + } + let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) + .map_err(|_| ClientError::Configuration("bearer is not a valid HTTP credential"))?; + authorization.set_sensitive(true); + Ok(ControlClient { + client: http_client(authorization.clone())?, + authorization, + base_url, + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs index e72ebe13fc..f445706317 100644 --- a/crates/sqlbuild-orchestration/src/api/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -1,2 +1,4 @@ +pub(crate) mod client_response; +pub(crate) mod connect_client; pub mod openapi_document; pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index e7db400f75..c0ec5c75a9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -28,3 +28,10 @@ pub struct HeartbeatResponse { pub struct ApiErrorBody { pub error: String, } + +#[derive(Clone, Debug)] +pub(crate) struct ControlClient { + pub(crate) client: reqwest::Client, + pub(crate) authorization: reqwest::header::HeaderValue, + pub(crate) base_url: reqwest::Url, +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs index cbac454f9b..79bfd1f284 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -2,17 +2,21 @@ use uuid::Uuid; +use crate::api::main::client_response::client_response; use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::runs::models::{RunChange, RunChangeRequest, RunExecution}; -use crate::worker::_helpers::response::response; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; impl WorkerClient { /// Fetch the launcher's immutable request, not the location's latest deployment. pub async fn execution(&self) -> Result { - let execution: RunExecution = - response(self.client.get(format!("{}execution", self.run_url))).await?; + let execution: RunExecution = client_response( + self.connection + .client + .get(format!("{}execution", self.run_url)), + ) + .await?; if execution.claim.request.run_id != self.run_id || execution.claim.claim_id != self.claim_id { @@ -31,8 +35,9 @@ impl WorkerClient { claim_id: Some(self.claim_id), change, }; - let result: MutationResponse = response( - self.client + let result: MutationResponse = client_response( + self.connection + .client .post(format!("{}events", self.run_url)) .json(&request), ) @@ -42,8 +47,12 @@ impl WorkerClient { /// Run independently from warehouse execution; cancellation is a cooperative request. pub async fn heartbeat(&self) -> Result { - let result: HeartbeatResponse = - response(self.client.post(format!("{}heartbeat", self.run_url))).await?; + let result: HeartbeatResponse = client_response( + self.connection + .client + .post(format!("{}heartbeat", self.run_url)), + ) + .await?; Ok(result.cancel_requested) } } diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs index a8afb1fce8..25beaf886c 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs @@ -5,7 +5,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use crate::runs::models::RunExecution; -use crate::worker::_helpers::response::http_client; use crate::worker::errors::WorkerError; use crate::worker::models::{WorkerClient, WorkerSession}; @@ -62,13 +61,13 @@ pub(crate) async fn start_session( } }; runtime.block_on(async move { - let heartbeat = match http_client(client.authorization.clone()) { - Ok(http) => WorkerClient { - client: http, + let heartbeat = match client.connection.independent() { + Ok(connection) => WorkerClient { + connection, ..client }, Err(error) => { - let _ = ready.send(Err(error)); + let _ = ready.send(Err(WorkerError::from(error))); return; } }; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs index 1bbe75bead..9ec828d671 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod client; pub(crate) mod heartbeat; -pub(crate) mod response; pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs deleted file mode 100644 index 0790ae0288..0000000000 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/response.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Do not echo response bodies: server diagnostics can contain user configuration. - -use std::time::Duration; - -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; -use serde::de::DeserializeOwned; - -use crate::worker::errors::WorkerError; - -/// Each runtime owns its connection pool; sharing pools could tie heartbeat IO to build work. -pub(crate) fn http_client(authorization: HeaderValue) -> Result { - let mut headers = HeaderMap::new(); - headers.insert(AUTHORIZATION, authorization); - Ok(reqwest::Client::builder() - .default_headers(headers) - .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(30)) - .connect_timeout(Duration::from_secs(10)) - .build()?) -} - -pub(crate) async fn response( - request: reqwest::RequestBuilder, -) -> Result { - let response = request.send().await.map_err(reqwest::Error::without_url)?; - if !response.status().is_success() { - return Err(WorkerError::Rejected(response.status())); - } - Ok(response.json().await.map_err(reqwest::Error::without_url)?) -} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs index e5e62c936a..49240df632 100644 --- a/crates/sqlbuild-orchestration/src/worker/errors.rs +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -15,3 +15,13 @@ pub enum WorkerError { #[error("run cancellation was requested before execution")] Cancelled, } + +impl From for WorkerError { + fn from(error: crate::api::errors::ClientError) -> Self { + match error { + crate::api::errors::ClientError::Configuration(message) => Self::Configuration(message), + crate::api::errors::ClientError::Transport(error) => Self::Transport(error), + crate::api::errors::ClientError::Rejected(status) => Self::Rejected(status), + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs index cf90ad449f..15a7a4d081 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -1,45 +1,21 @@ //! Bound worker requests have deadlines and must never follow credential-bearing redirects. -use reqwest::header::HeaderValue; use uuid::Uuid; -use crate::worker::_helpers::response::http_client; +use crate::api::main::connect_client::connect_client; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; -fn worker_url(server_url: &str, run_id: Uuid) -> Result { - let mut url = reqwest::Url::parse(server_url) - .map_err(|_| WorkerError::Configuration("server URL must be an absolute HTTP URL"))?; - if !matches!(url.scheme(), "http" | "https") - || url.host_str().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.query().is_some() - || url.fragment().is_some() - { - return Err(WorkerError::Configuration( - "server URL must use HTTP or HTTPS without credentials, query or fragment", - )); - } - let prefix = url.path().trim_end_matches('/'); - url.set_path(&format!("{prefix}/api/runs/{run_id}/")); - Ok(url) -} - /// Connect to an API origin or path prefix using the launcher's run-scoped bearer. pub fn connect_worker( server_url: &str, bearer: &str, (run_id, claim_id): (Uuid, Uuid), ) -> Result { - let run_url = worker_url(server_url, run_id)?; - let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) - .map_err(|_| WorkerError::Configuration("run bearer is not a valid HTTP credential"))?; - authorization.set_sensitive(true); - let client = http_client(authorization.clone())?; + let connection = connect_client(server_url, bearer)?; + let run_url = connection.endpoint(&["api", "runs", &run_id.to_string(), ""])?; Ok(WorkerClient { - client, - authorization, + connection, run_url, run_id, claim_id, diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs index 02e05c2e89..81799ca75f 100644 --- a/crates/sqlbuild-orchestration/src/worker/models.rs +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -5,12 +5,12 @@ use std::sync::atomic::AtomicBool; use uuid::Uuid; +use crate::api::models::ControlClient; use crate::runs::models::RunExecution; #[derive(Clone, Debug)] pub struct WorkerClient { - pub(crate) client: reqwest::Client, - pub(crate) authorization: reqwest::header::HeaderValue, + pub(crate) connection: ControlClient, pub(crate) run_url: reqwest::Url, pub(crate) run_id: Uuid, pub(crate) claim_id: Uuid, From 1e50ac1d91b2cbf7c8197ca0cf8d5029197a025d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:51:29 +0000 Subject: [PATCH 013/138] feat: publish retained compiler snapshots through the control API --- Cargo.lock | 1 + .../src/snapshots/main/mod.rs | 1 + .../snapshots/main/publish_remote_snapshot.rs | 23 +++++ .../tests/api/test_contract.rs | 20 +++- crates/sqlbuild-python/Cargo.toml | 1 + .../_helpers/compiled_project/snapshot.rs | 18 +++- src/sqlbuild/_native.pyi | 1 + .../compiler/compiled_project/helpers.py | 17 +++- .../compiled_project/publication/__init__.py | 1 + .../publication/_test_types.py | 18 ++++ .../compiled_project/publication/helpers.py | 95 ++++++++++++++++++ .../publication/test_publication.py | 97 +++++++++++++++++++ .../test_orchestration_snapshot.py | 16 +-- 13 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py diff --git a/Cargo.lock b/Cargo.lock index 92db47c581..060ae30b32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2647,6 +2647,7 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "stacker", + "tokio", ] [[package]] diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 88c2bf1eba..8411aaf37d 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,3 +1,4 @@ pub mod current_snapshot; +pub mod publish_remote_snapshot; pub mod publish_snapshot; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs new file mode 100644 index 0000000000..681cc0f61e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs @@ -0,0 +1,23 @@ +//! A code location sends retained compiled data over HTTP and never opens the control store. + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; + +pub async fn publish_remote_snapshot( + server_url: &str, + bearer: &str, + input: &SnapshotInput, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &input.location_id.to_string(), + "targets", + &input.target, + "snapshots", + ])?; + client_response(connection.client.post(url).json(input)).await +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 53d9903ae5..bea9439da0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -4,6 +4,7 @@ use std::time::Duration; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; +use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; @@ -15,6 +16,7 @@ use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, }; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::store::main::create_location::create_location; @@ -107,16 +109,26 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); let input = snapshot(location_id); + let published = publish_remote_snapshot(&api.url, &location_token.bearer, &input) + .await + .expect("publish retained native data"); let response = client - .post(&path) + .get(&path) .bearer_auth(&location_token.bearer) - .json(&input) .send() .await - .expect("publish through API"); + .expect("read published snapshot through API"); assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers()["cache-control"], "no-store"); - let published: SnapshotRecord = response.json().await.expect("snapshot response"); + drop(response); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &input).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &snapshot(other)).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); let response = client .post(&path) .bearer_auth(&location_token.bearer) diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index be0cc75d7c..3b066a9b33 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -47,6 +47,7 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true stacker.workspace = true +tokio.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index b0b0d878fe..fa850b69dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -3,11 +3,12 @@ use std::collections::BTreeMap; use std::path::PathBuf; -use pyo3::exceptions::PyValueError; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; @@ -29,6 +30,21 @@ impl NativeOrchestrationSnapshot { py.detach(|| serde_json::to_string(&self.input)) .map_err(|error| PyValueError::new_err(error.to_string())) } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + let record = result.map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } } /// Compile catalog nodes from their owners, never from Python model or graph projections. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3452f57cab..78a8f577b2 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1742,6 +1742,7 @@ class NativeOrchestrationSnapshot: """A compiled data-only publication retained natively until transport.""" def to_json(self) -> str: ... + def publish(self, server_url: str, bearer: str) -> str: ... def build_orchestration_snapshot( project: CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index f351592f4d..024306d0a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -8,7 +8,8 @@ from collections.abc import Callable, Iterator from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from uuid import UUID import pytest @@ -47,6 +48,20 @@ if TYPE_CHECKING: from sqlbuild._native import CompileInputsView +METADATA: dict[str, Any] = { + "location_id": str(UUID(int=1)), + "target": "development", + "generation": 1, + "git_revision": None, + "git_dirty": False, + "git_author": None, + "image_id": None, + "environment_hash": "test-environment", + "sqb_version": "not-the-installed-version", + "nodes": {}, + "documents": {}, +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py new file mode 100644 index 0000000000..75debee041 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py @@ -0,0 +1 @@ +"""Retained compiler snapshots published through the real native control API.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py new file mode 100644 index 0000000000..8c7301d86d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -0,0 +1,18 @@ +"""Publication cases and isolated server credentials.""" + +from dataclasses import dataclass +from typing import NamedTuple + + +class SnapshotApi(NamedTuple): + url: str + location_id: str + bearer: str + + +@dataclass(frozen=True) +class SnapshotPublicationTestCase: + description: str + target: str + expected_generation: int + expected_nodes: frozenset[str] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py new file mode 100644 index 0000000000..cbc2554451 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -0,0 +1,95 @@ +"""A standalone native API and real SQLite store, with test-only credential provisioning.""" + +from __future__ import annotations + +import hashlib +import json +import os +import secrets +import sqlite3 +import subprocess +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen +from uuid import uuid4 + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotApi, +) + + +def api_address(process: subprocess.Popen[bytes], log: Path) -> str: + """Wait for bounded native startup, never publishing an unready address.""" + deadline: float = time.monotonic() + 10 + while "Control API ready: " not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, "Native API did not become ready" + time.sleep(0.02) + return "http://" + log.read_text().partition("Control API ready: ")[2].splitlines()[0] + + +@contextmanager +def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: + """Only this fixture seeds the control database; the code location uses HTTP exclusively.""" + directory.mkdir() + log: Path = directory / "server.log" + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", "--no-scheduler"], + cwd=directory, + env=environment, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + url: str = api_address(process, log) + location_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_locations (location_id,name,created_by,created_at) " + "VALUES (?, 'orders', 'test-admin', ?)", + (location_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,location_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + location_id, + json.dumps({"kind": "location", "location_id": location_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + yield SnapshotApi(url=url, location_id=location_id, bearer=bearer) + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: + """Observe the public API instead of inspecting stored snapshot implementation details.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/{quote(target, safe='')}/snapshots", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + result: dict[str, Any] = json.load(response) + return result diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py new file mode 100644 index 0000000000..ce5ca06abd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -0,0 +1,97 @@ +"""Real compile, native transport, authenticated server and durable publication.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotPublicationTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="ordinary target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="target requiring URL path encoding", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_retained_compiler_data_when_publishing_then_http_updates_atomically_and_rejections_keep_last_good( + test_case: SnapshotPublicationTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | { + "location_id": api.location_id, + "target": test_case.target, + } + first: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + record: dict[str, Any] = json.loads(first.publish(api.url, api.bearer)) + assert record["generation"] == 1 + assert record["location_id"] == api.location_id + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] + edited: dict[str, str] = TYPE_PROOF_FILES | { + "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" + ) + } + write_files(project_dir=project_dir, files=edited) + changed: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + next_snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + changed, json.dumps(metadata | {"generation": 2}), project_dir + ) + updated: dict[str, Any] = json.loads(next_snapshot.publish(api.url, api.bearer)) + assert updated["generation"] == test_case.expected_generation + assert updated["snapshot_id"] != record["snapshot_id"] + with pytest.raises(RuntimeError, match="HTTP 409"): + _ = first.publish(api.url, api.bearer) + with pytest.raises(RuntimeError, match="HTTP 401"): + _ = next_snapshot.publish(api.url, "invalid-credential") + current: dict[str, Any] = current_snapshot(api, test_case.target) + assert current["record"] == updated + assert "CAST(9 AS INTEGER)" in current["nodes"]["model:stg_orders"]["raw_sql"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index e53269d8a3..f32d0b2601 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -5,7 +5,6 @@ import json from pathlib import Path from typing import Any -from uuid import UUID import pytest @@ -24,25 +23,12 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, + METADATA, PARITY_FILES, TYPE_PROOF_FILES, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files -METADATA: dict[str, Any] = { - "location_id": str(UUID(int=1)), - "target": "development", - "generation": 1, - "git_revision": None, - "git_dirty": False, - "git_author": None, - "image_id": None, - "environment_hash": "test-environment", - "sqb_version": "not-the-installed-version", - "nodes": {}, - "documents": {}, -} - @pytest.mark.parametrize( "test_case", From bb56e96c6bb1e0ea2998584a6195490aad0ce456 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:08:00 +0000 Subject: [PATCH 014/138] feat: ingest scoped worker output through the control API --- Cargo.lock | 2 + .../src/api/_helpers/runs.rs | 21 ++- .../src/api/main/serve_api.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 3 +- .../src/runs/_helpers/mod.rs | 1 + .../src/runs/_helpers/output.rs | 80 ++++++++ .../src/runs/constants.rs | 3 + .../src/runs/main/mod.rs | 1 + .../src/runs/main/publish_run_output.rs | 36 ++++ .../sqlbuild-orchestration/src/runs/models.rs | 14 ++ .../src/worker/_helpers/client.rs | 17 +- crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/output/helpers.rs | 178 ++++++++++++++++++ .../tests/api/test_contract.rs | 33 ++++ .../tests/api/test_worker.rs | 12 ++ crates/sqlbuild-recall/Cargo.toml | 2 + crates/sqlbuild-recall/src/records/models.rs | 21 ++- 17 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/output.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/output/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index 060ae30b32..2246634aaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,11 +2658,13 @@ dependencies = [ "chrono", "rusqlite", "rustix", + "serde", "serde_json", "sha2", "sqlbuild-config", "sqlbuild-core", "tempfile", + "utoipa", ] [[package]] diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index ef93a4a15a..a0f39f3324 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -3,6 +3,7 @@ use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::{Extension, Json}; +use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; use crate::api::errors::ApiError; @@ -10,8 +11,9 @@ use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::runs::main::change_run::change_run; use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::publish_run_output::publish_run_output; use crate::runs::main::run_execution::run_execution; -use crate::runs::models::{RunAuthority, RunChangeRequest, RunExecution}; +use crate::runs::models::{OutputPublication, RunAuthority, RunChangeRequest, RunExecution}; use crate::store::models::ControlStore; #[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), @@ -68,6 +70,23 @@ pub(crate) async fn heartbeat( })) } +#[utoipa::path(post, path = "/api/runs/{run_id}/output", params(("run_id" = Uuid, Path)), + request_body = Vec, + responses((status = 200, body = OutputPublication), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn output( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(chunks): Json>, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_output(&store, (run_id, claim_id), &chunks).await?, + )) +} + fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { if let CredentialScope::Run { run_id: permitted, diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 99927066e8..2dd444bef6 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -25,6 +25,7 @@ pub async fn serve_api( .route("/api/runs/{run_id}/execution", get(runs::execution)) .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .route("/api/runs/{run_id}/output", post(runs::output)) .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); let application = protected diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c0ec5c75a9..6120833396 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -10,7 +10,8 @@ use crate::api::_helpers::{runs, snapshots}; snapshots::current, runs::execution, runs::report, - runs::heartbeat + runs::heartbeat, + runs::output ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index 75a1590516..5bfdb31c61 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -2,5 +2,6 @@ pub(crate) mod changes; pub(crate) mod claims; pub(crate) mod events; pub(crate) mod nodes; +pub(crate) mod output; pub(crate) mod projection; pub(crate) mod publish_change; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs new file mode 100644 index 0000000000..d6daa4602a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs @@ -0,0 +1,80 @@ +//! Resolve output ownership from the physical claim and normalize Recall input before append. + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputChunk, RecallRow, RowEnvelope}; +use uuid::Uuid; + +use crate::runs::constants::{OUTPUT_BATCH_BYTES, OUTPUT_BATCH_CHUNKS}; +use crate::runs::models::OutputMetadata; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn output_envelope( + transaction: ControlTransaction, + (run_id, claim_id): (Uuid, Uuid), +) -> Result<(ControlTransaction, RowEnvelope), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT l.claimed_at, s.sqb_version FROM sqb_run_liveness l \ + JOIN sqb_runs r ON r.run_id=l.run_id JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE l.run_id=$1 AND l.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let metadata: OutputMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("output does not belong to an active run claim".into()) + })?)?; + Ok(( + transaction, + RowEnvelope { + invocation_id: format!("worker/{claim_id}"), + invocation_started_at: metadata + .claimed_at + .to_rfc3339_opts(SecondsFormat::Micros, true), + producer_version: metadata.sqb_version, + }, + )) +} + +/// Keep MAX(sequence) + 1 representable for Recall completeness accounting. +pub(crate) fn output_rows( + envelope: &RowEnvelope, + chunks: &[OutputChunk], +) -> Result, ControlError> { + if chunks.len() > OUTPUT_BATCH_CHUNKS + || chunks.iter().map(|chunk| chunk.data.len()).sum::() > OUTPUT_BATCH_BYTES + { + return Err(ControlError::Invalid( + "output batch exceeds its limit".into(), + )); + } + let mut rows = Vec::with_capacity(chunks.len()); + for chunk in chunks { + let mut chunk = chunk.clone(); + let mut integers = vec![chunk.sequence, chunk.byte_offset, chunk.dropped_before]; + if let Some(omitted) = chunk.omitted { + integers.extend([omitted.bytes, omitted.sequences]); + if !chunk.data.is_empty() || omitted.sequences == 0 { + return Err(ControlError::Invalid( + "invalid output omission marker".into(), + )); + } + } + if chunk.sequence >= i64::MAX as u64 + || integers.iter().any(|value| i64::try_from(*value).is_err()) + { + return Err(ControlError::Invalid( + "output counter is out of range".into(), + )); + } + chunk.occurred_at = DateTime::parse_from_rfc3339(&chunk.occurred_at) + .map_err(|_| ControlError::Invalid("invalid output timestamp".into()))? + .to_utc() + .to_rfc3339_opts(SecondsFormat::Micros, true); + rows.push(output_row(envelope, &chunk)); + } + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 801d9c9edb..f53d82c1b5 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -5,3 +5,6 @@ pub(crate) const NODE_RUNNING: &str = "running"; pub(crate) const HEARTBEAT_TIMEOUT_SECONDS: i64 = 45; pub(crate) const MONITOR_BATCH_SIZE: usize = 100; + +pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; +pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index ffceba28a4..46615363df 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -4,6 +4,7 @@ pub mod heartbeat_run; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; +pub mod publish_run_output; pub(crate) mod queue_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs new file mode 100644 index 0000000000..35633a3a0e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -0,0 +1,36 @@ +//! Exact-byte Recall output is namespaced by the authenticated physical claim, never a body ID. + +use sqlbuild_recall::records::models::OutputChunk; +use uuid::Uuid; + +use crate::runs::_helpers::output::{output_envelope, output_rows}; +use crate::runs::models::OutputPublication; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// The active-claim check and the complete batch commit share a serialized write transaction. +pub async fn publish_run_output( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), + chunks: &[OutputChunk], +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, envelope) = output_envelope(transaction, (run_id, claim_id)).await?; + let mut published = OutputPublication::default(); + for row in output_rows(&envelope, chunks)? { + let outcome; + (transaction, outcome) = append_recall(transaction, &row).await?; + if outcome.inserted { + published.inserted += 1; + } else if outcome.conflict.is_some() { + published.conflicts += 1; + } else { + published.duplicates += 1; + } + } + commit(transaction).await?; + Ok(published) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 636dc6891a..bf98fc240b 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -264,3 +264,17 @@ pub(crate) struct ChangingRun { pub(crate) cancel_requested_at: Option>, pub(crate) node_counts_json: BTreeMap, } + +/// Differing retries retain conflict evidence while preserving the first output bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct OutputPublication { + pub inserted: u32, + pub duplicates: u32, + pub conflicts: u32, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OutputMetadata { + pub(crate) claimed_at: DateTime, + pub(crate) sqb_version: String, +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs index 79bfd1f284..813c99c3ac 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -1,10 +1,11 @@ //! A worker's authenticated HTTP requests carry only its own run and claim. +use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; use crate::api::main::client_response::client_response; use crate::api::models::{HeartbeatResponse, MutationResponse}; -use crate::runs::models::{RunChange, RunChangeRequest, RunExecution}; +use crate::runs::models::{OutputPublication, RunChange, RunChangeRequest, RunExecution}; use crate::worker::errors::WorkerError; use crate::worker::models::WorkerClient; @@ -45,6 +46,20 @@ impl WorkerClient { Ok(result.applied) } + /// Retry the same chunks after an uncertain response; differing retries never overwrite bytes. + pub async fn publish_output( + &self, + chunks: &[OutputChunk], + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}output", self.run_url)) + .json(chunks), + ) + .await?) + } + /// Run independently from warehouse execution; cancellation is a cooperative request. pub async fn heartbeat(&self) -> Result { let result: HeartbeatResponse = client_response( diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 14ce5705f3..9c411b15ae 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -1,5 +1,7 @@ #[path = "api/helpers.rs"] mod helpers; +#[path = "api/output/helpers.rs"] +mod output_helpers; #[path = "runs/helpers.rs"] mod run_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs new file mode 100644 index 0000000000..6c628f05de --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -0,0 +1,178 @@ +//! Worker output contract helpers inspect durable bytes on both databases. + +use serde_json::json; +use sqlbuild_orchestration::runs::models::OutputPublication; +use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::models::WorkerClient; +use sqlbuild_recall::records::models::{OmittedOutput, OutputChunk, OutputStream}; +use uuid::Uuid; + +pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputChunk { + OutputChunk { + sequence, + stream, + occurred_at: "2026-01-02T03:04:05Z".into(), + data: data.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + } +} + +async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { + let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + let connection = rusqlite::Connection::open(path).expect("inspect SQLite bytes"); + let mut statement = connection.prepare(sql).expect("output query"); + return statement + .query_map([invocation], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("stored rows") + .collect::>() + .expect("decode bytes"); + }; + sqlx_core::query_as::query_as(sql) + .bind(invocation) + .fetch_all(pool) + .await + .expect("inspect PostgreSQL bytes") +} + +pub(crate) async fn output_contract(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let invocation = format!("worker/{claim_id}"); + let first = chunk(0, OutputStream::Stdout, b"orders\0\xff\n"); + let warning = chunk(1, OutputStream::Stderr, b"warning\xc3\x28"); + let omission = OutputChunk { + omitted: Some(OmittedOutput { + bytes: 43, + sequences: 2, + }), + byte_offset: first.data.len() as u64, + dropped_before: 1, + ..chunk(3, OutputStream::Stdout, b"") + }; + let chunks = [first.clone(), warning.clone(), omission]; + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("publish exact bytes"), + OutputPublication { + inserted: 3, + duplicates: 0, + conflicts: 0 + } + ); + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("retry accepted output"), + OutputPublication { + inserted: 0, + duplicates: 3, + conflicts: 0 + } + ); + let mut equivalent = first.clone(); + equivalent.occurred_at = "2026-01-02T04:04:05+01:00".into(); + assert_eq!( + worker + .publish_output(&[equivalent]) + .await + .expect("equivalent timestamp retry") + .duplicates, + 1 + ); + let mut conflict = first.clone(); + conflict.data = b"different".to_vec(); + assert_eq!( + worker + .publish_output(&[conflict]) + .await + .expect("record retry conflict") + .conflicts, + 1 + ); + assert_eq!( + stored_bytes(store, &invocation).await, + vec![ + (0, first.data.clone()), + (1, warning.data.clone()), + (3, vec![]) + ] + ); + let conflicts = read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("durable conflicts"); + assert_eq!(conflicts.len(), 1); + let counts = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("output accounting"); + assert_eq!(counts[0]["stdout_bytes"], json!(first.data.len())); + assert_eq!(counts[0]["stderr_bytes"], json!(warning.data.len())); + assert_eq!(counts[0]["truncated_bytes"], json!(43)); + + let valid = chunk(4, OutputStream::Stdout, b"must roll back"); + let mut invalid = chunk(5, OutputStream::Stderr, b"invalid timestamp"); + invalid.occurred_at = "not-a-time".into(); + for rejected in [ + vec![valid.clone(), invalid], + vec![OutputChunk { + sequence: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + sequence: i64::MAX as u64, + ..valid.clone() + }], + vec![OutputChunk { + byte_offset: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + dropped_before: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + omitted: Some(OmittedOutput { + bytes: 1, + sequences: 0, + }), + ..valid.clone() + }], + vec![valid.clone(); 257], + vec![chunk(4, OutputStream::Stdout, &vec![0; 1_048_577])], + ] { + assert!(matches!( + worker.publish_output(&rejected).await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + assert_eq!( + stored_bytes(store, &invocation).await.len(), + 3, + "invalid batches are atomic" + ); + assert_eq!( + worker + .publish_output(&vec![first; 256]) + .await + .expect("chunk count boundary") + .duplicates, + 256 + ); + let boundary = chunk(4, OutputStream::Stdout, &vec![0; 1_048_576]); + assert_eq!( + worker + .publish_output(&[boundary]) + .await + .expect("byte count boundary") + .inserted, + 1 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index bea9439da0..31668c3a0b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -374,6 +374,39 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .status(), StatusCode::FORBIDDEN ); + for (output_run, credential) in [ + (Uuid::now_v7(), &worker.bearer), + (run_id, &location_token.bearer), + ] { + assert_eq!( + client + .post(format!("{}/api/runs/{output_run}/output", api.url)) + .bearer_auth(credential) + .json(&json!([])) + .send() + .await + .expect("output scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + let mut forged = json!(crate::output_helpers::chunk( + 0, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"forged" + )); + forged["invocation_id"] = json!("scheduler/another-run"); + assert_eq!( + client + .post(format!("{}/api/runs/{run_id}/output", api.url)) + .bearer_auth(&worker.bearer) + .json(&json!([forged])) + .send() + .await + .expect("body cannot choose its output namespace") + .status(), + StatusCode::UNPROCESSABLE_ENTITY + ); let cancellation = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::CancelRequested { diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 5cb27bd833..83fa332103 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -23,6 +23,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use uuid::Uuid; use crate::helpers::{start_api, stop_api}; +use crate::output_helpers::{chunk, output_contract}; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; @@ -140,6 +141,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a let session = start_worker(&worker, &baseline) .await .expect("verified start"); + output_contract(store, &worker, identity.1).await; assert_eq!(session.execution.claim, claim, "{}", test_case.description); assert_eq!( start_worker(&worker, &baseline) @@ -206,6 +208,16 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a ) .await .expect("finish"); + assert!(matches!( + worker + .publish_output(&[chunk( + 5, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"late" + )]) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); let runs = list_runs(store, (location, "development"), 10) .await .expect("completed"); diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index cfabda69e4..8b54b4542e 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -11,10 +11,12 @@ rust-version.workspace = true blake3.workspace = true chrono.workspace = true rusqlite.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +utoipa.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index e99a1cf280..a7798e3e5c 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -62,7 +62,10 @@ pub struct RowEnvelope { } /// A process output stream. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum OutputStream { Stdout, Stderr, @@ -89,7 +92,8 @@ impl OutputStream { } /// One exact-byte output chunk. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] pub struct OutputChunk { /// Per invocation, from 0, shared by both streams. pub sequence: u64, @@ -105,7 +109,18 @@ pub struct OutputChunk { } /// Output of one stream that was not recorded because it exceeded `max_output_bytes`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(deny_unknown_fields)] pub struct OmittedOutput { pub bytes: u64, /// Sequences this gap accounts for, its own included. From 23cd632adad96f402912f63a016c947c4a148243 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:23:59 +0000 Subject: [PATCH 015/138] feat: bootstrap audited locations and credentials natively Add privileged native administration with JSON results, bounded location credentials and atomic security audit. Detect lost result delivery without claiming committed mutations rolled back. --- Cargo.lock | 1 + crates/sqlbuild-cli/Cargo.toml | 1 + .../sqlbuild-cli/src/argv/command_table.json | 199 ++++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 36 +++- .../src/native_commands/_helpers/execution.rs | 21 +- .../src/native_commands/constants.rs | 1 + .../src/native_commands/main/_run_admin.rs | 45 ++++ .../src/native_commands/main/_run_control.rs | 15 +- .../src/native_commands/main/mod.rs | 1 + .../main/run_control_invocation.rs | 57 +++-- .../src/native_commands/models.rs | 10 + crates/sqlbuild-cli/tests/native_commands.rs | 4 + .../tests/native_commands/admin/helpers.rs | 55 +++++ .../tests/native_commands/admin/test_admin.rs | 182 ++++++++++++++++ .../tests/native_commands/admin/test_types.rs | 14 ++ .../tests/native_commands/test_control.rs | 47 +++++ .../tests/native_commands/worker/helpers.rs | 16 +- .../main/create_location.rs | 28 ++- .../src/credentials/main/mod.rs | 1 + .../src/store/main/mod.rs | 1 - crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/test_contract.rs | 2 +- .../tests/api/test_worker.rs | 2 +- .../tests/credentials/helpers.rs | 18 ++ .../tests/credentials/locations/helpers.rs | 24 +++ .../tests/credentials/test_contract.rs | 9 +- .../sqlbuild-orchestration/tests/platform.rs | 2 + .../tests/platform/test_watchdog.rs | 2 +- crates/sqlbuild-orchestration/tests/runs.rs | 2 + .../tests/runs/test_claims.rs | 2 +- .../tests/runs/test_coalesce.rs | 2 +- .../tests/runs/test_faults.rs | 2 +- .../tests/runs/test_lifecycle.rs | 2 +- .../tests/runs/test_monitor.rs | 2 +- .../sqlbuild-orchestration/tests/scheduler.rs | 2 + .../tests/scheduler/ticks/helpers.rs | 2 +- .../tests/scheduler/ticks/test_loop.rs | 2 +- .../tests/scheduler/ticks/test_triggers.rs | 2 +- .../sqlbuild-orchestration/tests/snapshots.rs | 2 + .../tests/snapshots/helpers.rs | 2 +- 40 files changed, 760 insertions(+), 60 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs rename crates/sqlbuild-orchestration/src/{store => credentials}/main/create_location.rs (57%) create mode 100644 crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index 2246634aaa..7f825f1ccc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2490,6 +2490,7 @@ name = "sqlbuild-cli" version = "0.186.2" dependencies = [ "blake3", + "chrono", "ctrlc", "getrandom 0.4.3", "reqwest", diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 686080a599..fd9d3be80e 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,6 +17,7 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +chrono.workspace = true getrandom.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ec58647f34..c5dad55e42 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,200 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "admin", + "parser": { + "prog": "sqb admin", + "description": "Privileged local administration using direct control database access. Results are JSON; issued bearer credentials appear only once on stdout.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "parsers", + "dest": "admin_command", + "nargs": "A...", + "required": true, + "group": 0, + "parsers": [ + { + "name": "create-location", + "parser": { + "prog": "sqb admin create-location", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "location label; identity is assigned by the control store", + "group": 0 + } + ] + } + }, + { + "name": "issue-location-token", + "parser": { + "prog": "sqb admin issue-location-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, + { + "name": "revoke-token", + "parser": { + "prog": "sqb admin revoke-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TOKEN_ID", + "required": true, + "help": "credential UUID, never its bearer secret", + "group": 0 + } + ] + } + } + ], + "choices_help": [ + { + "dest": "create-location", + "metavar": "create-location", + "help": "register a location with an atomic security audit record" + }, + { + "dest": "issue-location-token", + "metavar": "issue-location-token", + "help": "issue a bounded location credential, returned once as JSON" + }, + { + "dest": "revoke-token", + "metavar": "revoke-token", + "help": "revoke a credential by UUID" + } + ] + } + ] + } + }, { "name": "server", "parser": { @@ -7719,6 +7913,11 @@ } ], "choices_help": [ + { + "dest": "admin", + "metavar": "admin", + "help": "administer locations and credentials through privileged database access" + }, { "dest": "server", "metavar": "server", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 21c6342413..8ccde9b2a8 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -3,7 +3,9 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; +use chrono::{DateTime, Utc}; use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; use crate::argv::models::{ArgvOutcome, Namespace, Value}; @@ -15,7 +17,8 @@ use crate::native_commands::constants::{ }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, + ControlAdmin, CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, + SkillsOptions, }; use sqlbuild_core::cost_command::constants::DESCENDING_ORDER; @@ -39,6 +42,37 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } +pub(crate) fn control_admin(namespace: &Namespace) -> Result { + let value = text(namespace, "admin_subject").unwrap_or_default(); + if text(namespace, "admin_command").as_deref() == Some("create-location") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + return Ok(ControlAdmin::CreateLocation(value)); + } + let id = Uuid::parse_str(&value) + .map_err(|_| ControlError::Invalid("a valid location or token UUID is required".into()))?; + match text(namespace, "admin_command").as_deref() { + Some("issue-location-token") => { + let expiry = DateTime::parse_from_rfc3339( + &text(namespace, "admin_expires_at").unwrap_or_default(), + ) + .map_err(|_| ControlError::Invalid("--expires-at must be an RFC3339 timestamp".into()))? + .with_timezone(&Utc); + if expiry <= Utc::now() { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + Ok(ControlAdmin::IssueLocationToken(id, expiry)) + } + Some("revoke-token") => Ok(ControlAdmin::RevokeToken(id)), + _ => Err(ControlError::Invalid( + "an administration command is required".into(), + )), + } +} + pub(crate) fn control_address( namespace: &Namespace, standalone: bool, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index ac2f27678e..0a84181be4 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -5,16 +5,20 @@ use std::path::{Path, PathBuf}; use std::time::SystemTime; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; use tokio::sync::watch; use tokio::task::JoinSet; -use crate::argv::models::Stream; +use crate::argv::models::{Namespace, Stream}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::decision::control_database; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::project_location; @@ -29,6 +33,21 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +/// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. +pub(crate) async fn open_control_store( + namespace: &Namespace, +) -> Result { + let url = control_database(namespace)?; + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + let store = open_store(&url).await?; + migrate_store(&store).await?; + drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + Ok(store) +} + async fn shutdown_signal() -> Result<(), ControlError> { #[cfg(unix)] { diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 7f993a2320..74c0358e56 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -43,6 +43,7 @@ pub(crate) const INIT_COMMAND: &str = "init"; pub(crate) const SKILLS_COMMAND: &str = "skills"; pub(crate) const SERVER_COMMAND: &str = "server"; pub(crate) const SCHEDULER_COMMAND: &str = "scheduler"; +pub(crate) const ADMIN_COMMAND: &str = "admin"; pub(crate) const RULES_COMMAND: &str = "rules"; pub(crate) const RULES_LIST_ACTION: &str = "list"; pub(crate) const RULES_SHOW_ACTION: &str = "show"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs new file mode 100644 index 0000000000..06b21d9aea --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -0,0 +1,45 @@ +//! Direct database ownership authorizes bootstrap, not an untrusted HTTP request or supplied role. + +use serde_json::{Value, json}; +use sqlbuild_orchestration::credentials::main::create_location::create_location; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::control_admin; +use crate::native_commands::_helpers::execution::open_control_store; +use crate::native_commands::models::ControlAdmin; + +pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let action = control_admin(namespace)?; + let store = open_control_store(namespace).await?; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + match action { + ControlAdmin::CreateLocation(name) => { + let id = create_location(&store, &name, &context).await?; + Ok(json!({"location_id": id, "name": name})) + } + ControlAdmin::IssueLocationToken(location_id, expires_at) => { + let token = issue_token( + &store, + &context, + CredentialScope::Location { location_id }, + expires_at, + ) + .await?; + Ok(json!(token)) + } + ControlAdmin::RevokeToken(token_id) => { + let revoked = revoke_token(&store, &context, token_id).await?; + Ok(json!({"token_id": token_id, "revoked": revoked})) + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 06f74ae5df..1f39c05c8b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -5,14 +5,12 @@ use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::open_store::open_store; use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, control_database, flag}; -use crate::native_commands::_helpers::execution::supervise_control_roles; +use crate::native_commands::_helpers::decision::{control_address, flag}; +use crate::native_commands::_helpers::execution::{open_control_store, supervise_control_roles}; use crate::native_commands::_helpers::terminal::write_output; async fn stopped(mut shutdown: watch::Receiver) { @@ -28,14 +26,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let url = control_database(namespace)?; - drop(write_output(&[( - Stream::Stderr, - "Opening control store...\n", - )])); - let store = open_store(&url).await?; - migrate_store(&store).await?; - drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + let store = open_control_store(namespace).await?; let (stop, shutdown) = watch::channel(false); let (_changes, changes) = watch::channel(()); let mut tasks = tokio::task::JoinSet::new(); diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index c52a4a29a2..e93969bdc4 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,4 +1,5 @@ pub(super) mod _cost_report_options; +pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; pub(crate) mod parse_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs index 699407935b..6ff78bf47b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -5,12 +5,16 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{flag, text}; use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; -use crate::native_commands::constants::{SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_control::run_control; pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { let command = text(namespace, "command")?; - if !matches!(command.as_str(), SERVER_COMMAND | SCHEDULER_COMMAND) { + if !matches!( + command.as_str(), + SERVER_COMMAND | SCHEDULER_COMMAND | ADMIN_COMMAND + ) { return None; } if flag(namespace, "control_openapi") { @@ -42,22 +46,45 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { return Some(1); } }; - Some( - match runtime.block_on(run_control(namespace, command == SCHEDULER_COMMAND)) { - Ok(()) => { + let admin = command == ADMIN_COMMAND; + let role = if admin { "administration" } else { "service" }; + let result = runtime.block_on(async { + if admin { + run_admin(namespace).await.map(Some) + } else { + run_control(namespace, command == SCHEDULER_COMMAND) + .await + .map(|()| None) + } + }); + Some(match result { + Ok(output) => { + let lost = + output.and_then(|value| write_output(&[(Stream::Stdout, &format!("{value}\n"))])); + if admin && lost.is_some() { drop(write_output(&[( Stream::Stderr, - "Control service stopped successfully\n", + "Control administration JSON result delivery failed; changes may already be committed\n", )])); - 0 + return Some(1); } - Err(error) => { - drop(write_output(&[( - Stream::Stderr, - &format!("Control service failed: {error}\n"), - )])); - 1 + let exit_code = finish_lost_stdout(lost.as_ref(), 0); + if exit_code == 0 { + let message = if admin { + "Control administration completed successfully\n" + } else { + "Control service stopped successfully\n" + }; + drop(write_output(&[(Stream::Stderr, message)])); } - }, - ) + exit_code + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control {role} failed: {error}\n"), + )])); + 1 + } + }) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f328de4e67..9ae73c2995 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -2,8 +2,18 @@ use std::path::PathBuf; +use chrono::{DateTime, Utc}; +use uuid::Uuid; + use crate::argv::models::Namespace; +/// Privileged local operations require direct ownership of the control database. +pub(crate) enum ControlAdmin { + CreateLocation(String), + IssueLocationToken(Uuid, DateTime), + RevokeToken(Uuid), +} + /// Parsing can finish an invocation before package discovery or any Python handoff. #[derive(Debug)] pub(crate) enum NativeParse { diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index e93113875b..208f02150b 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -1,5 +1,9 @@ +#[path = "native_commands/admin/helpers.rs"] +mod admin_helpers; #[path = "native_commands/helpers.rs"] mod helpers; +#[path = "native_commands/admin/test_admin.rs"] +mod test_admin; #[path = "native_commands/test_control.rs"] mod test_control; #[path = "native_commands/test_types.rs"] diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs new file mode 100644 index 0000000000..ae7e64d9c4 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -0,0 +1,55 @@ +//! Run the isolated native executable; never include returned bearer bytes in assertion messages. + +use std::process::Command; + +use serde_json::Value; + +use crate::test_types::TestControl; + +pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { + let output = Command::new( + control + .directory + .path() + .join(format!("sqb{}", std::env::consts::EXE_SUFFIX)), + ) + .arg("admin") + .args(arguments) + .current_dir(control.directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .output() + .expect("native administration process"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration completed successfully\n" + ); + serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") +} + +pub(crate) fn audit_actions(control: &TestControl, bearer: &str) -> Vec { + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect control audit"); + let mut query = connection.prepare("SELECT action, actor, granting_role, scope_json || before_json || after_json FROM sqb_security_audit ORDER BY occurred_at, event_id") + .expect("audit records"); + query + .query_map([], |row| { + let action: String = row.get(0)?; + let actor: String = row.get(1)?; + let role: String = row.get(2)?; + let payload: String = row.get(3)?; + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert!(!payload.contains(bearer)); + Ok(action) + }) + .expect("read audit") + .collect::>() + .expect("decode audit") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs new file mode 100644 index 0000000000..70e03d2970 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -0,0 +1,182 @@ +//! Native bootstrap credentials authorize real publication and stop working after revocation. + +#[path = "test_types.rs"] +mod test_types; + +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use uuid::Uuid; + +use crate::admin_helpers::{administer, audit_actions}; +use crate::helpers::{api_ready, start_control}; +#[cfg(unix)] +use crate::test_admin::test_types::AdminDeliveryTestCase; +use crate::test_admin::test_types::AdminTestCase; + +#[test] +fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() + { + let test_cases = [AdminTestCase { + description: "native operator bootstrap preserves labels and scopes credentials", + location_name: "orders / café", + expected_name: "orders / café", + expected_actions: [ + "location.created", + "location.created", + "token.issued", + "token.revoked", + ], + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let first = administer(&control, &["create-location", test_case.location_name]); + let second = administer(&control, &["create-location", "customers"]); + let location_id = + Uuid::parse_str(first["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + let other_id = Uuid::parse_str(second["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + assert_ne!(location_id, other_id); + assert_eq!( + first["name"], test_case.expected_name, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-location-token", + &location_id.to_string(), + "--expires-at", + &expiry, + ], + ); + let bearer = credential["bearer"].as_str().expect("one-time bearer"); + assert_eq!(credential["identity"]["scope"]["kind"], "location"); + assert_eq!( + credential["identity"]["scope"]["location_id"], + location_id.to_string() + ); + let mut snapshot = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::new(), + documents: BTreeMap::new(), + }; + let published = publish_remote_snapshot(&url, bearer, &snapshot) + .await + .expect("operator-issued credential publishes"); + assert_eq!(published.location_id, location_id); + snapshot.location_id = other_id; + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + )); + snapshot.location_id = location_id; + snapshot.generation = 2; + let token_id = credential["identity"]["token_id"] + .as_str() + .expect("credential ID"); + for expected in [true, false] { + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], expected); + assert_eq!(revoked["token_id"], token_id); + } + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + let mut actions = audit_actions(&control, bearer); + actions.sort(); + assert_eq!( + actions, test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[cfg(unix)] +#[test] +fn given_a_committed_admin_operation_when_json_output_is_lost_then_all_response_sizes_fail_without_claiming_rollback() + { + let test_cases = [ + AdminDeliveryTestCase { + description: "small response", + name_bytes: 8, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "Python compatibility drop window", + name_bytes: 5000, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "large response", + name_bytes: 9000, + expected_exit: 1, + expected_rows: (1, 1), + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated administration directory"); + let binary = crate::helpers::isolated_binary(directory.path()); + let output = std::process::Command::new(binary) + .args([ + "admin", + "create-location", + &"x".repeat(test_case.name_bytes), + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout( + std::fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("failing output device"), + ) + .output() + .expect("native administration"); + assert_eq!( + output.status.code(), + Some(test_case.expected_exit), + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration JSON result delivery failed; changes may already be committed\n", + "{}", + test_case.description + ); + let connection = + rusqlite::Connection::open(directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect committed operation"); + let rows: (i64, i64) = connection.query_row("SELECT (SELECT COUNT(*) FROM sqb_locations), (SELECT COUNT(*) FROM sqb_security_audit)", [], |row| Ok((row.get(0)?, row.get(1)?))).expect("location and audit remain committed"); + assert_eq!(rows, test_case.expected_rows, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs new file mode 100644 index 0000000000..62aac53bee --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -0,0 +1,14 @@ +pub(crate) struct AdminTestCase { + pub(crate) description: &'static str, + pub(crate) location_name: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 4], +} + +#[cfg(unix)] +pub(crate) struct AdminDeliveryTestCase { + pub(crate) description: &'static str, + pub(crate) name_bytes: usize, + pub(crate) expected_exit: i32, + pub(crate) expected_rows: (i64, i64), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 85a2e4e44c..4f4ffc1f3c 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -21,6 +21,53 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb ", expected_stderr: "", }, + CommandTestCase { + description: "native privileged administration help", + arguments: &["admin", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank location rejected before store initialization", + arguments: &["admin", "create-location", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: location name is required\n", + }, + CommandTestCase { + description: "invalid credential ID rejected before store initialization", + arguments: &["admin", "revoke-token", "not-a-uuid"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: a valid location or token UUID is required\n", + }, + CommandTestCase { + description: "invalid expiry rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "tomorrow", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --expires-at must be an RFC3339 timestamp\n", + }, + CommandTestCase { + description: "expired credential rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "2000-01-01T00:00:00Z", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: token expiry must be in the future\n", + }, CommandTestCase { description: "invalid port is rejected before store initialization", arguments: &["server", "--port", "65536"], diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 2460bfd6ab..c951d675f2 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use std::time::{Duration, SystemTime}; +use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; @@ -16,7 +17,6 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_orchestration::worker::main::start_worker::start_worker; @@ -34,7 +34,13 @@ pub(crate) async fn start_claimed_worker( let store = open_store(path.to_str().expect("test database path")) .await .expect("native control store"); - let location_id = create_location(&store, "orders", "test-admin") + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(&store, "orders", &context) .await .expect("location"); let baseline = SnapshotInput { @@ -93,12 +99,6 @@ pub(crate) async fn start_claimed_worker( .expect("claim") .expect("queued work"); let identity = (run_id, claim.claim_id); - let context = AuditContext { - actor: "test-admin".into(), - granting_role: "admin".into(), - request_id: Uuid::now_v7(), - ip_address: None, - }; let token = issue_token( &store, &context, diff --git a/crates/sqlbuild-orchestration/src/store/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs similarity index 57% rename from crates/sqlbuild-orchestration/src/store/main/create_location.rs rename to crates/sqlbuild-orchestration/src/credentials/main/create_location.rs index 075de8a140..88d4d8d148 100644 --- a/crates/sqlbuild-orchestration/src/store/main/create_location.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs @@ -1,7 +1,10 @@ //! Create a server-owned location identity independent of its mutable label. +use serde_json::json; use uuid::Uuid; +use crate::credentials::_helpers::audit::audit; +use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -12,19 +15,30 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn create_location( store: &ControlStore, name: &str, - actor: &str, + context: &AuditContext, ) -> Result { - if name.trim().is_empty() || actor.trim().is_empty() { - return Err(ControlError::Invalid( - "location name and actor are required".into(), - )); + if name.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); } let mut transaction = begin(store, TransactionMode::Write).await?; let id = Uuid::now_v7(); let now = transaction.now; (transaction, _) = query(transaction, "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", - vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(actor.into()), Parameter::Time(now)]).await?; - commit(transaction).await?; + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; + commit( + audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "location.created"), + ( + json!({"location_id": id}), + json!(null), + json!({"name": name}), + ), + ) + .await?, + ) + .await?; Ok(id) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index 6b63069d7b..49e61796cf 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,4 +1,5 @@ pub mod authenticate_token; +pub mod create_location; pub mod create_principal; pub mod issue_token; pub mod revoke_token; diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index d22e12f725..de522badbc 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -2,7 +2,6 @@ pub(crate) mod append_control_event; pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; -pub mod create_location; pub(crate) mod decode_row; pub mod migrate_store; pub mod open_store; diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 9c411b15ae..543523edd6 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -1,5 +1,7 @@ #[path = "api/helpers.rs"] mod helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "api/output/helpers.rs"] mod output_helpers; #[path = "runs/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 31668c3a0b..72ea9f093b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -19,11 +19,11 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 83fa332103..e488f61f11 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -15,7 +15,6 @@ use sqlbuild_orchestration::runs::models::{ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::worker::errors::WorkerError; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; @@ -23,6 +22,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use uuid::Uuid; use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; use crate::output_helpers::{chunk, output_contract}; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index e01295e856..f86c76460c 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -1,5 +1,23 @@ use sqlbuild_orchestration::store::models::ControlStore; +pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { + let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ + (SELECT COUNT(*) FROM sqb_security_audit WHERE action='location.created')"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect locations") + .query_row(sql, [], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("location and audit counts"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_one(pool) + .await + .expect("location and audit counts") +} + pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs new file mode 100644 index 0000000000..c6edcc2874 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -0,0 +1,24 @@ +//! Shared location fixtures use the same audited identity operation as operator bootstrap. + +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn create_location( + store: &ControlStore, + name: &str, + actor: &str, +) -> Result { + sqlbuild_orchestration::credentials::main::create_location::create_location( + store, + name, + &AuditContext { + actor: actor.into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 4c39180fad..3c3c62f087 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use chrono::{Duration, Utc}; use sqlbuild_orchestration::credentials::main::authenticate_token::authenticate_token; +use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; @@ -9,12 +10,11 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::stored_hashes; +use crate::helpers::{location_state, stored_hashes}; use crate::replica_helpers::second_store; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; @@ -69,6 +69,8 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .await .is_err() ); + assert!(create_location(store, "orders", &context).await.is_err()); + assert_eq!(location_state(&replica).await, (0, 0)); remove_fault(store, &fault).await; let principal_id = create_principal( store, @@ -77,9 +79,10 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the ) .await .expect("principal creation and audit rolled back together"); - let location_id = create_location(store, "orders", "test-admin") + let location_id = create_location(store, "orders", &context) .await .expect("location"); + assert_eq!(location_state(&replica).await, (1, 1)); let scopes = [ CredentialScope::Principal { principal_id }, CredentialScope::Location { location_id }, diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs index 7908cbf127..3243ae357c 100644 --- a/crates/sqlbuild-orchestration/tests/platform.rs +++ b/crates/sqlbuild-orchestration/tests/platform.rs @@ -1,5 +1,7 @@ #[path = "platform/helpers.rs"] mod helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 6954c6bfe0..e119342ea7 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -5,11 +5,11 @@ use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::PlatformEvent; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::helpers::{snapshot, wait_for_scans}; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 12d6c66d52..2ab841868c 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -6,6 +6,8 @@ mod fault_types; mod helpers; #[path = "store/leases/helpers.rs"] mod lease_helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "runs/monitor/helpers.rs"] mod monitor_helpers; #[path = "store/replicas/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 509232d7f2..fdf5e34d4e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -6,12 +6,12 @@ use sqlbuild_orchestration::runs::main::list_runs::list_runs; use sqlbuild_orchestration::runs::models::RunStatus; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; use crate::helpers::{queue, request, snapshot}; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index 8c0ea844dd..b38db00540 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -8,11 +8,11 @@ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; use sqlbuild_orchestration::scheduler::models::TickDecision; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::helpers::snapshot; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::CoalescedRunTestCase; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 6af9167fbb..d32ffd0472 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -9,7 +9,6 @@ use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeReq use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; @@ -17,6 +16,7 @@ use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::{ClaimFailureTestCase, TransactionFault}; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 0f02f31093..d06a397736 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -10,13 +10,13 @@ use sqlbuild_orchestration::runs::models::{ }; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::test_types::{RunLifecycleTestCase, TransactionFault}; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index c4847d7316..e567d9d25b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -17,7 +17,6 @@ use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; @@ -25,6 +24,7 @@ use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::helpers::{queue, request, snapshot}; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::monitor_helpers::{age_heartbeat, start_run}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs index 7142cff07b..774474ceef 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler.rs @@ -6,6 +6,8 @@ mod fault_types; mod helpers; #[path = "store/leases/helpers.rs"] mod lease_helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index fa51f828f5..1cbed7a105 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -12,13 +12,13 @@ use sqlbuild_orchestration::scheduler::models::TickOutcome; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index f9341cf027..fbe1a63283 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -6,11 +6,11 @@ use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::lease_helpers::expire_lease; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 4df7178b12..cf2cfb68ac 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -9,11 +9,11 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; +use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::ticks::helpers::snapshot; diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs index 58d1347746..13be1a8abc 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots.rs @@ -2,6 +2,8 @@ mod contract; #[path = "snapshots/helpers.rs"] mod helpers; +#[path = "credentials/locations/helpers.rs"] +mod location_helpers; #[path = "store/replicas/helpers.rs"] mod replica_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index b1533caba3..9030c6f077 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -7,12 +7,12 @@ use sqlbuild_orchestration::snapshots::main::current_snapshot::current_snapshot; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::create_location::create_location; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; +use crate::location_helpers::create_location; use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store_test_types::Backend; From 259f1d969c0985914f35b37b2b62a0c1028bbf88 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:59:45 +0000 Subject: [PATCH 016/138] fix: use a scalar expression in the DuckShop function fixture --- .../project/functions/sql/edge_cases/edge_next_quantity.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql index c4fa936dea..d49cb02a03 100644 --- a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql +++ b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql @@ -5,4 +5,4 @@ FUNCTION ( returns INTEGER, ); -SELECT quantity + 1 +quantity + 1 From 2904f549c9c4c47b158d118b08ca8ac2bdf69722 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:47:59 +0000 Subject: [PATCH 017/138] feat: compile authored jobs and cron schedules into snapshots Keep the public Python API limited to authoring records and native delegation. Resolve selectors and validate schedule data in Rust before publication. Preserve stable shorthand trigger names across cron edits. Add the explicitly approved narrow Python facade boundary entries without increasing movable Python or native crossing budgets. --- .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 21 ++- .../compiled_project/snapshot_declarations.rs | 121 ++++++++++++++++ .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_declarations.rs | 19 +++ python-boundary.toml | 15 ++ src/sqlbuild/_native.pyi | 2 + src/sqlbuild/orchestration.py | 8 ++ .../main/compile_snapshot.py | 23 +++ .../orchestration_declarations/models.py | 31 +++++ .../compiler/compiled_project/_test_types.py | 18 +++ .../test_orchestration_snapshot.py | 131 ++++++++++++++++++ 12 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs create mode 100644 src/sqlbuild/orchestration.py create mode 100644 src/sqlbuild/orchestration_declarations/main/compile_snapshot.py create mode 100644 src/sqlbuild/orchestration_declarations/models.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 1fb46a0eb0..67cb07e670 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod rows; pub(crate) mod schema_facts; pub(crate) mod snapshot; pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_models; pub(crate) mod snapshot_resources; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index fa850b69dd..7665d7a2d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -4,16 +4,21 @@ use std::collections::BTreeMap; use std::path::PathBuf; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::compile_declarations; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::records::fields::field; @@ -49,12 +54,14 @@ impl NativeOrchestrationSnapshot { /// Compile catalog nodes from their owners, never from Python model or graph projections. #[pyfunction] +#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] pub(crate) fn build_orchestration_snapshot( - py: Python<'_>, project: PyRef<'_, NativeProjectView>, metadata_json: &str, project_dir: PathBuf, + declarations: Option>>, ) -> PyResult { + let py = project.py(); let mut input: SnapshotInput = serde_json::from_str(metadata_json) .map_err(|error| PyValueError::new_err(error.to_string()))?; if !input.nodes.is_empty() { @@ -83,6 +90,16 @@ pub(crate) fn build_orchestration_snapshot( let owner = project.owner().bind(py).borrow(); input.nodes = model_nodes(py, &project, &owner, input.nodes)?; input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + drop(owner); + if let Some(declarations) = declarations { + validate_jobs(py, &declarations)?; + let authored = declarations + .iter() + .map(plain_value) + .collect::>>()?; + let graph = project.lineage_graph(py)?; + input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; + } Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs new file mode 100644 index 0000000000..bc2b0b3a9b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -0,0 +1,121 @@ +//! Compile authoring records against retained graph data before publication. + +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use serde::Deserialize; +use serde_json::Value; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; +use sqlbuild_orchestration::declarations::models::{ + JobDefinition, OnFailure, Schedule, TriggerDefinition, TriggerStatus, +}; +use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AuthoredSchedule { + cron: String, + timezone: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AuthoredJob { + name: String, + description: String, + select: String, + exclude: String, + overlap: Overlap, + on_failure: OnFailure, + concurrency_groups: Vec, + schedule: Option, + targets: Vec, +} + +pub(crate) fn compile_declarations( + mut input: SnapshotInput, + graph: &ProjectGraph, + declarations: Vec, +) -> PyResult { + if declarations.is_empty() { + return Ok(input); + } + if input.documents.contains_key("jobs") || input.documents.contains_key("triggers") { + return Err(PyValueError::new_err( + "authored declarations cannot overwrite snapshot job or trigger documents", + )); + } + let mut jobs: BTreeMap = BTreeMap::new(); + let mut triggers: BTreeMap = BTreeMap::new(); + let mut names: BTreeSet = BTreeSet::new(); + for declaration in declarations { + let job: AuthoredJob = serde_json::from_value(declaration) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !names.insert(job.name.clone()) { + return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); + } + if !job.targets.is_empty() && !job.targets.contains(&input.target) { + continue; + } + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + if job.select.trim().is_empty() { + return Err(PyValueError::new_err(format!( + "job {} needs a selection", + job.name + ))); + } + let selection: BTreeSet = + resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if let Some(schedule) = job.schedule { + triggers.insert( + job.name.clone(), + TriggerDefinition { + description: job.description.clone(), + job_key: job.name.clone(), + schedule: Schedule::Cron { + expression: schedule.cron, + timezone: schedule.timezone, + }, + starts_at: None, + default_status: TriggerStatus::Running, + catch_up: CatchUp::Latest, + }, + ); + } + jobs.insert( + job.name, + JobDefinition { + description: job.description, + selection, + settings: BTreeMap::new(), + overlap: job.overlap, + on_failure: job.on_failure, + concurrency_groups: job.concurrency_groups.into_iter().collect(), + }, + ); + } + for (key, document) in [ + ("jobs", serde_json::to_value(jobs)), + ("triggers", serde_json::to_value(triggers)), + ] { + input.documents.insert( + key.into(), + document.map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + } + snapshot_definitions(&input).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(input) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 637d9ef489..25adcc06b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod connections; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; +pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; pub(crate) mod registration; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs new file mode 100644 index 0000000000..c42491171b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -0,0 +1,19 @@ +//! The public authoring records enter native compilation here, never scheduler execution. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; + +pub(crate) fn validate_jobs( + py: Python<'_>, + declarations: &[Bound<'_, PyAny>], +) -> Result<(), PyErr> { + let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; + for declaration in declarations { + if !declaration.is_instance(&job)? { + return Err(PyTypeError::new_err( + "snapshot declarations must be sqlbuild.orchestration.Job objects", + )); + } + } + Ok(()) +} diff --git a/python-boundary.toml b/python-boundary.toml index ee2346f85d..b02d91a88d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -12,6 +12,21 @@ # data instead of projecting those public containers and immediately reading them back. remaining_lines = 0 +[[stays]] +path = "orchestration.py" +rule = 3 +note = "Public orchestration facade: re-exports only." + +[[stays]] +path = "orchestration_declarations/models.py" +rule = 3 +note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." + +[[stays]] +path = "orchestration_declarations/main/compile_snapshot.py" +rule = 3 +note = "Public declaration compiler: one-line delegation to build_orchestration_snapshot; no Python discovery, validation or snapshot logic." + [[stays]] path = "executor" rule = 1 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 78a8f577b2..6a4f400584 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1748,6 +1748,8 @@ def build_orchestration_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, + *, + declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... class NativeProjectView: diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py new file mode 100644 index 0000000000..94bb6c3ace --- /dev/null +++ b/src/sqlbuild/orchestration.py @@ -0,0 +1,8 @@ +"""Public authoring and snapshot compilation API for orchestration declarations.""" + +from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot +from sqlbuild.orchestration_declarations.models import Job, Schedule +from sqlbuild.orchestration_declarations.models import Job as job +from sqlbuild.orchestration_declarations.models import Schedule as schedule + +__all__ = ("Job", "Schedule", "compile_snapshot", "job", "schedule") diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py new file mode 100644 index 0000000000..2be661c7c1 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -0,0 +1,23 @@ +"""Delegate authoring-record compilation to the native retained snapshot owner.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration_declarations.models import Job + + +def compile_snapshot( + *, + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, + declarations: Sequence[Job] = (), +) -> NativeOrchestrationSnapshot: + """Compile authoring records and retained project facts into a data-only snapshot.""" + return build_orchestration_snapshot( + project, metadata_json, project_dir, declarations=declarations + ) diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py new file mode 100644 index 0000000000..278aa55335 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -0,0 +1,31 @@ +"""Authoring records only; selection, validation and scheduling belong to Rust.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + + +@dataclass(frozen=True, kw_only=True) +class Schedule: + """A wall-clock cron schedule attached to a job.""" + + cron: str + timezone: str + + +@dataclass(frozen=True, eq=False) +class Job: + """A named selection to execute, optionally on a wall-clock schedule.""" + + name: str + description: str = field(kw_only=True) + select: str = field(kw_only=True) + exclude: str = field(default="", kw_only=True) + overlap: Literal["coalesce", "queue", "allow"] = field(default="coalesce", kw_only=True) + on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = field( + default="skip_dependents", kw_only=True + ) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 6f5b1c5053..eba1f44a39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -214,3 +214,21 @@ class SnapshotEnvironmentTestCase: first_lockfile: str second_lockfile: str expected_environment_count: int + + +@dataclass(frozen=True) +class SnapshotScheduleTestCase: + description: str + timezone: str + expected_selection: tuple[str, ...] + + +@dataclass(frozen=True) +class SnapshotDeclarationRejectionTestCase: + description: str + job_description: str = "Orders." + select: str = "orders" + cron: str = "0 2 * * *" + timezone: str = "UTC" + copies: int = 1 + expected_message: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index f32d0b2601..e5787f636d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from dataclasses import replace from pathlib import Path from typing import Any @@ -16,10 +17,13 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration import Job, compile_snapshot, job, schedule from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, + SnapshotDeclarationRejectionTestCase, SnapshotEnvironmentTestCase, SnapshotRejectionTestCase, + SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, @@ -266,3 +270,130 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang assert changed == json.loads( build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotScheduleTestCase( + description="daylight saving zone", + timezone="Europe/London", + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="fixed offset zone", timezone="UTC", expected_selection=("model:orders",) + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_selection_and_cron( + tmp_path: Path, test_case: SnapshotScheduleTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + nightly: Job = job( + "nightly_orders", + description="Refresh order reporting.", + select="+orders", + exclude="stg_orders", + overlap="queue", + concurrency_groups=("order_reporting",), + schedule=schedule(cron="15 2 * * *", timezone=test_case.timezone), + targets=("development",), + ) + excluded: Job = replace(nightly, name="other_target", targets=("production",)) + first: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[nightly, excluded], + ).to_json() + ) + assert first["documents"]["jobs"] == { + "nightly_orders": { + "description": "Refresh order reporting.", + "selection": list(test_case.expected_selection), + "settings": {}, + "overlap": "queue", + "on_failure": "skip_dependents", + "concurrency_groups": ["order_reporting"], + } + } + trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] + assert trigger["job_key"] == "nightly_orders" + assert trigger["schedule"] == { + "kind": "cron", + "expression": "15 2 * * *", + "timezone": test_case.timezone, + } + edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) + second: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[edited], + ).to_json() + ) + assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() + assert second["documents"]["triggers"]["nightly_orders"]["schedule"]["expression"] == ( + "30 3 * * *" + ) + assert second["nodes"] == first["nodes"] + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDeclarationRejectionTestCase( + description="blank description", + job_description=" ", + expected_message="needs a description", + ), + SnapshotDeclarationRejectionTestCase( + description="blank selection", select="", expected_message="needs a selection" + ), + SnapshotDeclarationRejectionTestCase( + description="missing model", select="missing", expected_message="missing" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid cron", cron="not a cron", expected_message="cron" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate names", copies=2, expected_message="duplicate job orders" + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_declarations_when_compiling_then_native_validation_rejects_them( + tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + declarations: list[Job] = [ + job( + "orders", + description=test_case.job_description, + select=test_case.select, + schedule=schedule(cron=test_case.cron, timezone=test_case.timezone), + ) + ] * test_case.copies + with pytest.raises(ValueError, match=test_case.expected_message): + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + ) From 8b90eb5419d7b3806bb3e7fe49c141cd1d30c525 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:01:02 +0000 Subject: [PATCH 018/138] feat: discover orchestration declarations in code locations Collect exported jobs and nested factory containers natively, deduplicating aliases and terminating cyclic containers. Explicit empty declarations skip user module execution. Remove failed imports from the module cache so publication retries cannot reuse partial definitions. --- .../edge_cases/orchestration/failed_import.py | 3 + .../orchestration/edge_cases/definitions.py | 13 ++ crates/sqlbuild-discovery/src/constants.rs | 2 + .../src/python_paths/constants.rs | 5 +- .../tests/test_unsupported_python_paths.rs | 2 + .../src/entries/_helpers/user_code.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 15 ++- .../compiled_project/snapshot_declarations.rs | 64 +++++++--- .../native_host/orchestration_declarations.rs | 37 +++++- fensu.toml | 2 +- src/sqlbuild/_native_host.py | 4 + src/sqlbuild/_native_host.pyi | 3 + .../_helpers/filesystem/project_modules.py | 13 ++ .../main/compile_snapshot.py | 2 +- .../compiler/compiled_project/_test_types.py | 16 +++ .../test_orchestration_snapshot.py | 116 ++++++++++++++++++ 17 files changed, 279 insertions(+), 24 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/failed_import.py create mode 100644 benchmarks/duckshop/project/orchestration/edge_cases/definitions.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py new file mode 100644 index 0000000000..6483f0b2ab --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py @@ -0,0 +1,3 @@ +"""Negative fixture: a failed declaration import must fail every publication retry.""" + +raise RuntimeError("broken declaration") diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py new file mode 100644 index 0000000000..7d6e411ee8 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -0,0 +1,13 @@ +"""Exercise alias deduplication and cyclic factory containers during native discovery.""" + +from sqlbuild.orchestration import job, schedule + +edge_orders = job( + "edge_snapshot_orders", + description="Refresh only the synthetic edge-case models.", + select="tag:edge_case", + schedule=schedule(cron="15 2 * * *", timezone="UTC"), +) +edge_alias = edge_orders +edge_factory_results = [[edge_orders]] +edge_factory_results.append(edge_factory_results) diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 096580ebab..86a7a4dcd5 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -2,6 +2,8 @@ /// Root of SQL model files. pub const MODELS_DIRECTORY: &str = "models"; +/// Root of code-location orchestration declarations, never imported by the scheduler. +pub const ORCHESTRATION_DIRECTORY: &str = "orchestration"; /// Suffix of SQL files. pub const SQL_FILE_SUFFIX: &str = ".sql"; /// The folder below a resource tree that holds grouped scoped declarations. diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs index eacdb349f9..4483a9f780 100644 --- a/crates/sqlbuild-discovery/src/python_paths/constants.rs +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -1,7 +1,9 @@ //! The documented roots project Python may live under, and the directories the walk skips. +use crate::constants::ORCHESTRATION_DIRECTORY; + /// Top-level directories whose Python files are always supported. -pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ +pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 11] = [ "adapters", "materializations", "providers", @@ -12,6 +14,7 @@ pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ "enums", "constants", "event_exporters", + ORCHESTRATION_DIRECTORY, ]; /// `(root, language)` directory pairs whose Python files are supported. pub(crate) const LANGUAGE_PYTHON_ROOTS: [(&str, &str); 2] = diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs index 64ecd45aa6..9a4843ec44 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs @@ -11,6 +11,8 @@ fn given_project_python_when_listing_unsupported_paths_then_only_undocumented_ro files: &[ "adapter.py", "python/orders.py", + "orchestration/jobs.py", + "orchestration/nested/calendar.py", "functions/python/f.py", "macros/m.py", "models/_macros/m.py", diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index 7deafb99c8..51d567c4e1 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -64,6 +64,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, + HostEntry { + name: "load_orchestration_module", + kind: HostKind::Function, + edge: HostEdge::UserCode, + }, HostEntry { name: "module_functions", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 5c8482411a..057a1d4203 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -140,6 +140,7 @@ pub enum UserCodeHost { LoadProviderModule, LoadPythonNodeModule, LoadSinkModule, + LoadOrchestrationModule, ModuleFunctions, ModuleProviderClasses, ParameterAnnotations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 7665d7a2d6..cee2c105f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -13,10 +13,11 @@ use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::compile_declarations; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ + AuthoredJob, compile_declarations, discover_declarations, +}; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; -use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; @@ -91,12 +92,16 @@ pub(crate) fn build_orchestration_snapshot( input.nodes = model_nodes(py, &project, &owner, input.nodes)?; input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); - if let Some(declarations) = declarations { + let declarations = match declarations { + Some(declarations) => declarations, + None => discover_declarations(py, &project_dir)?, + }; + if !declarations.is_empty() { validate_jobs(py, &declarations)?; let authored = declarations .iter() - .map(plain_value) - .collect::>>()?; + .map(|declaration| declaration.extract::()) + .collect::>>()?; let graph = project.lineage_graph(py)?; input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index bc2b0b3a9b..af7683934d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -1,13 +1,18 @@ //! Compile authoring records against retained graph data before publication. use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::PyResult; -use serde::Deserialize; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ JobDefinition, OnFailure, Schedule, TriggerDefinition, TriggerStatus, @@ -15,22 +20,51 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] +use crate::bindings::_helpers::native_host::orchestration_declarations::module_jobs; + +/// Dedicated declaration modules are imported only in the code-location interpreter. +pub(crate) fn discover_declarations<'py>( + py: Python<'py>, + project_dir: &Path, +) -> PyResult>> { + let tree = ProjectTree::new(project_dir); + let files = py + .detach(|| { + rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir + && entry.name.ends_with(PYTHON_FILE_SUFFIX) + && entry.name.strip_suffix(PYTHON_FILE_SUFFIX) != Some(PYTHON_INIT_MODULE_STEM) + }) + }) + .map_err(|error| { + PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) + })?; + let mut declarations: Vec> = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for file in files { + for job in module_jobs(py, &tree.absolute(&file), project_dir)? { + if seen.insert(job.as_ptr() as usize) { + declarations.push(job); + } + } + } + Ok(declarations) +} + +#[derive(FromPyObject)] struct AuthoredSchedule { cron: String, timezone: String, } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct AuthoredJob { +#[derive(FromPyObject)] +pub(crate) struct AuthoredJob { name: String, description: String, select: String, exclude: String, - overlap: Overlap, - on_failure: OnFailure, + overlap: String, + on_failure: String, concurrency_groups: Vec, schedule: Option, targets: Vec, @@ -39,7 +73,7 @@ struct AuthoredJob { pub(crate) fn compile_declarations( mut input: SnapshotInput, graph: &ProjectGraph, - declarations: Vec, + declarations: Vec, ) -> PyResult { if declarations.is_empty() { return Ok(input); @@ -52,9 +86,7 @@ pub(crate) fn compile_declarations( let mut jobs: BTreeMap = BTreeMap::new(); let mut triggers: BTreeMap = BTreeMap::new(); let mut names: BTreeSet = BTreeSet::new(); - for declaration in declarations { - let job: AuthoredJob = serde_json::from_value(declaration) - .map_err(|error| PyValueError::new_err(error.to_string()))?; + for job in declarations { if !names.insert(job.name.clone()) { return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); } @@ -101,8 +133,10 @@ pub(crate) fn compile_declarations( description: job.description, selection, settings: BTreeMap::new(), - overlap: job.overlap, - on_failure: job.on_failure, + overlap: serde_json::from_value::(Value::String(job.overlap)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + on_failure: serde_json::from_value::(Value::String(job.on_failure)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, concurrency_groups: job.concurrency_groups.into_iter().collect(), }, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index c42491171b..6c3d086654 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,7 +1,42 @@ //! The public authoring records enter native compilation here, never scheduler execution. +use std::collections::HashSet; +use std::path::Path; + use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_host::entries::models::UserCodeHost; + +use crate::bindings::_helpers::native_host::host::entry; + +/// Only module execution crosses into Python; discovery and collection stay native. +pub(crate) fn module_jobs<'py>( + py: Python<'py>, + file_path: &Path, + project_dir: &Path, +) -> PyResult>> { + let options = PyDict::new(py); + options.set_item("file_path", file_path)?; + options.set_item("project_dir", project_dir)?; + let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; + let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; + let attributes = module.getattr("__dict__")?.cast_into::()?; + let mut pending: Vec> = attributes.values().iter().collect(); + let mut seen: HashSet = HashSet::new(); + let mut jobs: Vec> = Vec::new(); + while let Some(value) = pending.pop() { + if !seen.insert(value.as_ptr() as usize) { + continue; + } + if value.is_instance(&job_class)? { + jobs.push(value); + } else if value.is_instance_of::() || value.is_instance_of::() { + pending.extend(value.try_iter()?.collect::>>()?); + } + } + Ok(jobs) +} pub(crate) fn validate_jobs( py: Python<'_>, diff --git a/fensu.toml b/fensu.toml index 2338c3e56a..c06a29ffdd 100644 --- a/fensu.toml +++ b/fensu.toml @@ -229,7 +229,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] -symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_orchestration_module", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 9dca6ce419..fde2109986 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -167,6 +167,10 @@ "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "load_sink_module", ), + "load_orchestration_module": ( + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", + "load_orchestration_module", + ), "module_functions": ( "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "module_functions", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index c68e33d507..4ca6064432 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -130,6 +130,9 @@ from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_materialize_function as load_materialize_function, ) +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_orchestration_module as load_orchestration_module, +) from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_provider_module as load_provider_module, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index f28340d71b..1eef8c4987 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -244,6 +244,17 @@ def load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: ) +def load_orchestration_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Execute an orchestration module with the project's normal package imports.""" + + return _load_project_package_module( + file_path=file_path, + project_dir=project_dir, + file_label="orchestration", + error_type=PythonNodeDiscoveryError, + ) + + def load_materialize_function( *, file_path: Path, project_dir: Path ) -> Callable[..., object] | None: @@ -338,6 +349,8 @@ def _exec_project_module( sys.modules[module_name] = module spec.loader.exec_module(module) except Exception as error: + if sys.modules.get(module_name) is module: + sys.modules.pop(module_name, None) raise error_type( f"Failed to import {file_label} file {file_path.relative_to(project_dir)}: {error}" ) from error diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index 2be661c7c1..3cba43a00c 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job] = (), + declarations: Sequence[Job] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index eba1f44a39..6fd1933af6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -232,3 +232,19 @@ class SnapshotDeclarationRejectionTestCase: timezone: str = "UTC" copies: int = 1 expected_message: str = "" + + +@dataclass(frozen=True) +class SnapshotDiscoveryTestCase: + description: str + files: dict[str, str] + expected_jobs: tuple[str, ...] + declarations: tuple[()] | None = None + + +@dataclass(frozen=True) +class SnapshotDiscoveryRejectionTestCase: + description: str + files: dict[str, str] + expected_error: type[Exception] + expected_message: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index e5787f636d..f9e8c85502 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( NativeOrchestrationSnapshot, + PythonNodeDiscoveryError, build_compiled_project, build_orchestration_snapshot, discover_project_inputs, @@ -21,6 +22,8 @@ from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, + SnapshotDiscoveryRejectionTestCase, + SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, @@ -397,3 +400,116 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject project_dir=tmp_path, declarations=declarations, ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryTestCase( + description="relative imports aliases and factory containers", + files={ + "orchestration/shared.py": ( + "from sqlbuild.orchestration import job\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + ), + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "from .shared import staging\n" + "alias = staging\n" + "def factory():\n" + " return [job('daily', description='Daily orders.', select='orders')]\n" + "jobs = [factory(), staging]\n" + "jobs.append(jobs)\n" + ), + }, + expected_jobs=("daily", "staging"), + ), + SnapshotDiscoveryTestCase( + description="explicit empty declarations do not execute project files", + files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, + expected_jobs=(), + declarations=(), + ), + SnapshotDiscoveryTestCase( + description="DuckShop edge-case declaration containers", + files={ + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + }, + expected_jobs=("edge_snapshot_orders",), + ), + SnapshotDiscoveryTestCase( + description="a project without declarations has no job document", + files={}, + expected_jobs=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_modules_when_compiling_then_native_discovery_collects_jobs( + tmp_path: Path, test_case: SnapshotDiscoveryTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + publication: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=test_case.declarations, + ).to_json() + ) + assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs + for definition in publication["documents"].get("jobs", {}).values(): + assert set(definition["selection"]) <= publication["nodes"].keys() + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryRejectionTestCase( + description="distinct objects cannot share a job name", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" + ), + }, + expected_error=ValueError, + expected_message="duplicate job daily", + ), + SnapshotDiscoveryRejectionTestCase( + description="broken imports fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/failed_import.py" + ).read_text() + }, + expected_error=PythonNodeDiscoveryError, + expected_message="Failed to import orchestration file orchestration/jobs.py", + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapshot_is_returned( + tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + for _ in range(2): + with pytest.raises(test_case.expected_error, match=test_case.expected_message): + compile_snapshot( + project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path + ) From 067b438448b545540b0e8fb43512244391fc788d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:47:23 +0000 Subject: [PATCH 019/138] feat: validate authored orchestration job settings --- .../orchestration/conflicting_settings.py | 10 ++ .../orchestration/invalid_settings.py | 10 ++ .../orchestration/edge_cases/definitions.py | 2 + .../src/argv/_helpers/job_settings.rs | 128 ++++++++++++++++++ crates/sqlbuild-cli/src/argv/_helpers/mod.rs | 1 + .../src/argv/_helpers/normalization.rs | 40 +++--- crates/sqlbuild-cli/src/argv/constants.rs | 2 + .../src/argv/main/job_settings_arguments.rs | 32 +++++ crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + .../argv/tests/test_parse_cli_arguments.rs | 98 +++++++++++++- .../sqlbuild-cli/src/argv/tests/test_types.rs | 18 +++ .../compiled_project/snapshot_declarations.rs | 20 ++- .../native_host/orchestration_declarations.rs | 12 +- .../orchestration_declarations/models.py | 2 + .../compiler/compiled_project/_test_types.py | 2 + .../test_orchestration_snapshot.py | 82 ++++++++++- 16 files changed, 437 insertions(+), 23 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py create mode 100644 crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs create mode 100644 crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py new file mode 100644 index 0000000000..ae5a4c9dec --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: mutually exclusive build flags must fail before publication.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"load_sources": False, "reload": True}, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py new file mode 100644 index 0000000000..248b1b4ce1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: job settings must not override the frozen selection.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"select": "customers"}, +) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 7d6e411ee8..db9dff756a 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -6,6 +6,8 @@ "edge_snapshot_orders", description="Refresh only the synthetic edge-case models.", select="tag:edge_case", + settings={"run_tests": False}, + concurrency=2, schedule=schedule(cron="15 2 * * *", timezone="UTC"), ) edge_alias = edge_orders diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs new file mode 100644 index 0000000000..19bb1563e3 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -0,0 +1,128 @@ +//! Render one authored setting and check emitted options against the build parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::CONCURRENCY_DEST; +use crate::argv::models::{ActionKind, ActionSpec, Default, ParserSpec, Value}; + +pub(crate) fn setting_argument( + parser: &ParserSpec, + name: &str, + value: &Json, +) -> Result, String> { + if matches!( + name, + "help" + | "target" + | "project_dir" + | "select" + | "exclude" + | "select_file" + | "positional_select" + | "json" + | "json_output" + | "event_output" + | "manifest" + | "dbt_project_dir" + | "dbt_profiles_dir" + | "dbt_target" + | "dbt_target_path" + ) { + return Err(format!("job setting {name} is owned by the run request")); + } + let actions: Vec<&ActionSpec> = parser + .actions + .iter() + .filter(|action| action.dest.as_deref() == Some(name)) + .collect(); + let Some(action) = actions.first() else { + return Err(format!("unknown build setting {name}")); + }; + if matches!(action.kind, ActionKind::StoreTrue | ActionKind::StoreFalse) { + return boolean_argument(name, value, &actions); + } + let option = &action.option_strings[0]; + let text = setting_text(name, value, action)?; + if action + .choices + .as_ref() + .is_some_and(|choices| !choices.contains(&text)) + { + return Err(format!("job setting {name} is not an accepted choice")); + } + Ok(Some(format!("{option}={text}"))) +} + +fn boolean_argument( + name: &str, + value: &Json, + actions: &[&ActionSpec], +) -> Result, String> { + let action = actions[0]; + let flag = value + .as_bool() + .ok_or_else(|| format!("job setting {name} must be boolean"))?; + if let Some(chosen) = actions + .iter() + .find(|action| action.constant == Value::Bool(flag)) + { + return Ok(Some(chosen.option_strings[0].clone())); + } + if action.default != Default::Value(Value::Bool(flag)) + && !(action.default == Default::Suppress + && root_parser().actions.iter().any(|global| { + global.dest.as_deref() == Some(name) + && global.default == Default::Value(Value::Bool(flag)) + })) + { + return Err(format!("job setting {name} cannot be set to {flag}")); + } + Ok(None) +} + +fn setting_text(name: &str, value: &Json, action: &ActionSpec) -> Result { + match action.converter.as_deref() { + Some("int" | "_positive_integer") => { + let integer = value + .as_i64() + .ok_or_else(|| format!("job setting {name} must be an integer"))?; + if integer < 1 + && (name == CONCURRENCY_DEST + || action.converter.as_deref() == Some("_positive_integer")) + { + return Err(format!("job setting {name} must be positive")); + } + Ok(integer.to_string()) + } + Some("parse_cli_vars") => { + if !value.is_object() { + return Err(format!("job setting {name} must be a JSON object")); + } + Ok(value.to_string()) + } + _ => value + .as_str() + .ok_or_else(|| format!("job setting {name} must be text")) + .map(str::to_owned), + } +} + +pub(crate) fn validate_mutex(parser: &ParserSpec, arguments: &[String]) -> Result<(), String> { + let mut mutex_groups: BTreeMap = BTreeMap::new(); + for argument in arguments { + let option = argument + .split_once('=') + .map_or(argument.as_str(), |(option, _)| option); + if let Some(group) = parser.actions[parser.option_index[option]].mutex_group + && let Some(previous) = mutex_groups.insert(group, option) + { + return Err(format!( + "job settings {previous} and {option} are mutually exclusive" + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs index 22763ff51e..1d2d932e20 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod dbt_execution; pub(crate) mod formatting; +pub(crate) mod job_settings; pub(crate) mod namespace; pub(crate) mod normalization; pub(crate) mod parsing; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index 311d9ef353..b53bf9bf04 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -77,24 +77,10 @@ impl Run<'_> { .map_err(|message| self.error(parser, &message))?; namespace.set("concurrency", concurrency); } - if CURSOR_COMMANDS.contains(&command) { - let untyped: bool = ["start_cursor", "end_cursor"] - .iter() - .any(|name| namespace.value(name) != Value::None); - let typed: bool = [ - "start_cursor_ts", - "end_cursor_ts", - "start_cursor_int", - "end_cursor_int", - ] - .iter() - .any(|name| namespace.value(name) != Value::None); - if untyped && typed { - return Err(self.error( - parser, - "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", - )); - } + if CURSOR_COMMANDS.contains(&command) + && let Some(message) = cursor_error(|name| namespace.value(name) != Value::None) + { + return Err(self.error(parser, message)); } if command == RENAME_COMMAND && let Some(message) = rename_error(&namespace) @@ -156,6 +142,24 @@ impl Run<'_> { } } +/// The command line and authored job settings share the same cursor-family restriction. +pub(crate) fn cursor_error(present: impl Fn(&str) -> bool) -> Option<&'static str> { + let untyped = ["start_cursor", "end_cursor"] + .iter() + .any(|name| present(name)); + let typed = [ + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] + .iter() + .any(|name| present(name)); + (untyped && typed).then_some( + "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", + ) +} + /// The first rule `--since` and `--explain` break, on the commands that accept them. fn since_error(namespace: &Namespace) -> Option<&'static str> { match namespace.value("since") { diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index ef1b833fbb..3c079c4086 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -36,6 +36,8 @@ pub(crate) const MAXIMUM_BLANK_RUN: usize = 2; /// `CliNamespace` attributes the post-parse rules read and write. pub(crate) const COMMAND_DEST: &str = "command"; +pub(crate) const BUILD_COMMAND: &str = "build"; +pub(crate) const CONCURRENCY_DEST: &str = "concurrency"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_DEBUG_COMMAND: &str = "debug"; pub(crate) const DBT_PASSTHROUGH_COMMANDS: [&str; 4] = ["plan", "run", "build", "debug"]; diff --git a/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs new file mode 100644 index 0000000000..46d7b8ba46 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs @@ -0,0 +1,32 @@ +//! Validate frozen build settings against the same table as the command-line parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; +use crate::argv::_helpers::normalization::cursor_error; +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::BUILD_COMMAND; + +/// Produce argument tokens, never shell text; execution identity is owned by the run request. +pub fn job_settings_arguments(settings: &BTreeMap) -> Result, String> { + let parser = root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == BUILD_COMMAND) + .map(|(_, parser)| parser) + .ok_or_else(|| "build command is unavailable".to_owned())?; + let mut arguments: Vec = Vec::new(); + for (name, value) in settings { + if let Some(argument) = setting_argument(parser, name, value)? { + arguments.push(argument); + } + } + validate_mutex(parser, &arguments)?; + if let Some(message) = cursor_error(|name| settings.contains_key(name)) { + return Err(format!("job settings: {message}")); + } + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 0dabda8f2b..d456903240 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,3 +1,4 @@ pub mod cli_namespace_defaults; +pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index d47a6702c5..ab5d022216 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -1,6 +1,14 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::argv::main::job_settings_arguments::job_settings_arguments; use crate::argv::models::{ArgvOutcome, Value}; use crate::argv::tests::helpers::{parse, strings}; -use crate::argv::tests::test_types::{ExitTestCase, HelpTextTestCase, ParsedValueTestCase}; +use crate::argv::tests::test_types::{ + ExitTestCase, HelpTextTestCase, JobSettingsRejectionTestCase, JobSettingsTestCase, + ParsedValueTestCase, +}; #[test] fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values() { @@ -172,3 +180,91 @@ fn given_command_help_when_parsing_then_hidden_options_are_left_out() { assert_eq!(absent, test_case.expected_absent, "{help}"); } } + +#[test] +fn given_job_settings_when_validating_then_values_survive_the_build_parser() { + let test_cases = [ + JobSettingsTestCase { + description: "inverse flags and option-like values survive the build parser", + settings: BTreeMap::from([ + ("concurrency".into(), json!(3)), + ("load_sources".into(), json!(false)), + ("reload".into(), json!(false)), + ("run_tests".into(), json!(false)), + ("full_refresh".into(), json!(true)), + ("warehouse".into(), json!("--full-refresh")), + ]), + expected_values: vec![ + ("concurrency", Value::Int("3".into())), + ("load_sources", Value::Bool(false)), + ("reload", Value::Bool(false)), + ("run_tests", Value::Bool(false)), + ("full_refresh", Value::Bool(true)), + ("warehouse", Value::Str("--full-refresh".into())), + ], + expected_empty: false, + }, + JobSettingsTestCase { + description: "default-valued flags do not become explicit options", + settings: BTreeMap::from([ + ("run_tests".into(), json!(true)), + ("full_refresh".into(), json!(false)), + ("debug".into(), json!(false)), + ]), + expected_values: vec![], + expected_empty: true, + }, + ]; + for test_case in test_cases { + let settings = job_settings_arguments(&test_case.settings).expect(test_case.description); + assert_eq!( + settings.is_empty(), + test_case.expected_empty, + "{}", + test_case.description + ); + let mut arguments: Vec = vec!["build".to_owned()]; + arguments.extend(settings); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + let parsed = parse(&arguments).parsed.expect(test_case.description); + for (name, expected_value) in test_case.expected_values { + assert_eq!( + parsed.get(name), + Some(&expected_value), + "{}", + test_case.description + ); + } + } +} + +#[test] +fn given_conflicting_job_settings_when_validating_then_options_are_rejected() { + let test_cases = [ + JobSettingsRejectionTestCase { + description: "no-load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(false)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + JobSettingsRejectionTestCase { + description: "load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(true)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + ]; + for test_case in test_cases { + assert!( + job_settings_arguments(&test_case.settings) + .expect_err(test_case.description) + .contains(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 4452d1689e..793e4a87a3 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_core::dbt_cli::models::DbtParsedFlags; @@ -63,3 +66,18 @@ pub(super) struct DbtExecutionTestCase { pub(super) args: &'static [&'static str], pub(super) expected_parsed: Result, } + +/// Authored settings and the build namespace or validation failure they produce. +pub(super) struct JobSettingsTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_values: Vec<(&'static str, Value)>, + pub(super) expected_empty: bool, +} + +/// Mutually exclusive authored options rejected before a snapshot is published. +pub(super) struct JobSettingsRejectionTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_error: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index af7683934d..4e2fce5b69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use sqlbuild_discovery::tree::main::rglob::rglob; @@ -20,7 +21,9 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::native_host::orchestration_declarations::module_jobs; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + authored_setting, module_jobs, +}; /// Dedicated declaration modules are imported only in the code-location interpreter. pub(crate) fn discover_declarations<'py>( @@ -65,6 +68,10 @@ pub(crate) struct AuthoredJob { exclude: String, overlap: String, on_failure: String, + #[pyo3(from_py_with = authored_setting)] + settings: Value, + #[pyo3(from_py_with = authored_setting)] + concurrency: Value, concurrency_groups: Vec, schedule: Option, targets: Vec, @@ -93,6 +100,15 @@ pub(crate) fn compile_declarations( if !job.targets.is_empty() && !job.targets.contains(&input.target) { continue; } + let mut settings: BTreeMap = serde_json::from_value(job.settings) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !job.concurrency.is_null() { + if settings.contains_key("concurrency") { + return Err(PyValueError::new_err("job concurrency is declared twice")); + } + settings.insert("concurrency".into(), job.concurrency); + } + job_settings_arguments(&settings).map_err(PyValueError::new_err)?; let tokens = |value: &str| { value .split_whitespace() @@ -132,7 +148,7 @@ pub(crate) fn compile_declarations( JobDefinition { description: job.description, selection, - settings: BTreeMap::new(), + settings, overlap: serde_json::from_value::(Value::String(job.overlap)) .map_err(|error| PyValueError::new_err(error.to_string()))?, on_failure: serde_json::from_value::(Value::String(job.on_failure)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 6c3d086654..76ee58e258 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -3,13 +3,23 @@ use std::collections::HashSet; use std::path::Path; -use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::native_host::host::entry; +/// JSON authoring values cross once; the interpreter rejects circular user containers safely. +pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + serde_json::from_str(&text).map_err(|error| PyValueError::new_err(error.to_string())) +} + /// Only module execution crosses into Python; discovery and collection stay native. pub(crate) fn module_jobs<'py>( py: Python<'py>, diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 278aa55335..e3496353b1 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -26,6 +26,8 @@ class Job: on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = field( default="skip_dependents", kw_only=True ) + settings: dict[str, object] = field(default_factory=dict, kw_only=True) + concurrency: int | None = field(default=None, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 6fd1933af6..1744939e31 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -230,6 +230,8 @@ class SnapshotDeclarationRejectionTestCase: select: str = "orders" cron: str = "0 2 * * *" timezone: str = "UTC" + settings: dict[str, object] = field(default_factory=dict) + concurrency: int | None = None copies: int = 1 expected_message: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index f9e8c85502..58c60b9f7a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -304,6 +304,8 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select select="+orders", exclude="stg_orders", overlap="queue", + settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, + concurrency=4, concurrency_groups=("order_reporting",), schedule=schedule(cron="15 2 * * *", timezone=test_case.timezone), targets=("development",), @@ -321,7 +323,12 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select "nightly_orders": { "description": "Refresh order reporting.", "selection": list(test_case.expected_selection), - "settings": {}, + "settings": { + "concurrency": 4, + "run_tests": False, + "vars": {"batch": 3}, + "warehouse": "reporting", + }, "overlap": "queue", "on_failure": "skip_dependents", "concurrency_groups": ["order_reporting"], @@ -373,6 +380,57 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select SnapshotDeclarationRejectionTestCase( description="duplicate names", copies=2, expected_message="duplicate job orders" ), + SnapshotDeclarationRejectionTestCase( + description="unknown setting", + settings={"concurreny": 4}, + expected_message="unknown build setting concurreny", + ), + SnapshotDeclarationRejectionTestCase( + description="selection cannot escape its pinned run", + settings={"select": "customers"}, + expected_message="owned by the run request", + ), + SnapshotDeclarationRejectionTestCase( + description="zero concurrency", + concurrency=0, + expected_message="must be positive", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not concurrency", + concurrency=True, + expected_message="concurrency must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="integer is not a boolean flag", + settings={"full_refresh": 1}, + expected_message="must be boolean", + ), + SnapshotDeclarationRejectionTestCase( + description="integer strings are not typed settings", + settings={"concurrency": "3"}, + expected_message="must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="variables need an object", + settings={"vars": ["orders"]}, + expected_message="must be a JSON object", + ), + SnapshotDeclarationRejectionTestCase( + description="mutually exclusive load settings", + settings={"load_sources": False, "reload": True}, + expected_message="mutually exclusive", + ), + SnapshotDeclarationRejectionTestCase( + description="conflicting cursor families", + settings={"start_cursor": "2", "end_cursor_int": "4"}, + expected_message="cannot be combined", + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate concurrency declarations", + concurrency=3, + settings={"concurrency": 4}, + expected_message="declared twice", + ), ], ids=lambda case: case.description, ) @@ -390,6 +448,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "orders", description=test_case.job_description, select=test_case.select, + settings=test_case.settings, + concurrency=test_case.concurrency, schedule=schedule(cron=test_case.cron, timezone=test_case.timezone), ) ] * test_case.copies @@ -496,6 +556,26 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=PythonNodeDiscoveryError, expected_message="Failed to import orchestration file orchestration/jobs.py", ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop settings cannot replace the selection", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="owned by the run request", + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop mutually exclusive settings fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="mutually exclusive", + ), ], ids=lambda case: case.description, ) From 8958e37ad3956fe54d8377a2b5fdba929acc79e9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:53:57 +0000 Subject: [PATCH 020/138] feat: administer principal credentials from native CLI --- .../sqlbuild-cli/src/argv/command_table.json | 112 ++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 24 +++- .../src/native_commands/main/_run_admin.rs | 17 ++- .../src/native_commands/models.rs | 4 +- .../tests/native_commands/admin/test_admin.rs | 120 +++++++++++++++++- .../tests/native_commands/admin/test_types.rs | 7 + .../tests/native_commands/test_control.rs | 7 + 7 files changed, 276 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index c5dad55e42..ec7cee1ac8 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -162,6 +162,59 @@ ] } }, + { + "name": "create-principal", + "parser": { + "prog": "sqb admin create-principal", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "principal label; creation grants no permissions", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--kind" + ], + "dest": "admin_principal_kind", + "choices": [ + "person", + "service_account" + ], + "default": "person", + "help": "identity kind (default: person)", + "group": 1 + } + ] + } + }, { "name": "issue-location-token", "parser": { @@ -211,6 +264,55 @@ ] } }, + { + "name": "issue-principal-token", + "parser": { + "prog": "sqb admin issue-principal-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "PRINCIPAL_ID", + "required": true, + "help": "principal UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, { "name": "revoke-token", "parser": { @@ -257,11 +359,21 @@ "metavar": "create-location", "help": "register a location with an atomic security audit record" }, + { + "dest": "create-principal", + "metavar": "create-principal", + "help": "create a person or service account without granting permissions" + }, { "dest": "issue-location-token", "metavar": "issue-location-token", "help": "issue a bounded location credential, returned once as JSON" }, + { + "dest": "issue-principal-token", + "metavar": "issue-principal-token", + "help": "issue a bounded principal credential, returned once as JSON" + }, { "dest": "revoke-token", "metavar": "revoke-token", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 8ccde9b2a8..4055146ffc 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -4,6 +4,7 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -44,16 +45,27 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { pub(crate) fn control_admin(namespace: &Namespace) -> Result { let value = text(namespace, "admin_subject").unwrap_or_default(); - if text(namespace, "admin_command").as_deref() == Some("create-location") { + let action = text(namespace, "admin_command"); + if action.as_deref() == Some("create-location") { if value.trim().is_empty() { return Err(ControlError::Invalid("location name is required".into())); } return Ok(ControlAdmin::CreateLocation(value)); } + if action.as_deref() == Some("create-principal") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let kind = match text(namespace, "admin_principal_kind").as_deref() { + Some("service_account") => PrincipalKind::ServiceAccount, + _ => PrincipalKind::Person, + }; + return Ok(ControlAdmin::CreatePrincipal(value, kind)); + } let id = Uuid::parse_str(&value) .map_err(|_| ControlError::Invalid("a valid location or token UUID is required".into()))?; - match text(namespace, "admin_command").as_deref() { - Some("issue-location-token") => { + match action.as_deref() { + Some("issue-location-token" | "issue-principal-token") => { let expiry = DateTime::parse_from_rfc3339( &text(namespace, "admin_expires_at").unwrap_or_default(), ) @@ -64,7 +76,11 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result CredentialScope::Principal { principal_id: id }, + _ => CredentialScope::Location { location_id: id }, + }; + Ok(ControlAdmin::IssueToken(scope, expiry)) } Some("revoke-token") => Ok(ControlAdmin::RevokeToken(id)), _ => Err(ControlError::Invalid( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 06b21d9aea..8929365faf 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -2,9 +2,10 @@ use serde_json::{Value, json}; use sqlbuild_orchestration::credentials::main::create_location::create_location; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -27,14 +28,12 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { - let token = issue_token( - &store, - &context, - CredentialScope::Location { location_id }, - expires_at, - ) - .await?; + ControlAdmin::CreatePrincipal(name, kind) => { + let id = create_principal(&store, &context, (&name, kind)).await?; + Ok(json!({"principal_id": id, "name": name, "kind": kind})) + } + ControlAdmin::IssueToken(scope, expires_at) => { + let token = issue_token(&store, &context, scope, expires_at).await?; Ok(json!(token)) } ControlAdmin::RevokeToken(token_id) => { diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 9ae73c2995..df6d08a534 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use uuid::Uuid; use crate::argv::models::Namespace; @@ -10,7 +11,8 @@ use crate::argv::models::Namespace; /// Privileged local operations require direct ownership of the control database. pub(crate) enum ControlAdmin { CreateLocation(String), - IssueLocationToken(Uuid, DateTime), + CreatePrincipal(String, PrincipalKind), + IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), } diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index 70e03d2970..389afeb33b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -15,7 +15,7 @@ use crate::admin_helpers::{administer, audit_actions}; use crate::helpers::{api_ready, start_control}; #[cfg(unix)] use crate::test_admin::test_types::AdminDeliveryTestCase; -use crate::test_admin::test_types::AdminTestCase; +use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase}; #[test] fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() @@ -117,6 +117,124 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope } } +#[test] +fn given_native_principal_administration_when_issuing_credentials_then_identity_is_audited_without_granting_location_permissions() + { + let test_cases = [ + PrincipalAdminTestCase { + description: "a person credential has no implicit location authority", + kind: "person", + expected_name: "orders operator / café", + expected_actions: ["principal.created", "token.issued", "token.revoked"], + }, + PrincipalAdminTestCase { + description: "a service account credential has no implicit location authority", + kind: "service_account", + expected_name: "orders automation", + expected_actions: ["principal.created", "token.issued", "token.revoked"], + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let principal = administer( + &control, + &[ + "create-principal", + test_case.expected_name, + "--kind", + test_case.kind, + ], + ); + let principal_id = Uuid::parse_str( + principal["principal_id"] + .as_str() + .expect(test_case.description), + ) + .expect(test_case.description); + assert_eq!( + principal["name"], test_case.expected_name, + "{}", + test_case.description + ); + assert_eq!( + principal["kind"], test_case.kind, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-principal-token", + &principal_id.to_string(), + "--expires-at", + &expiry, + ], + ); + assert_eq!( + credential["identity"]["scope"]["kind"], "principal", + "{}", + test_case.description + ); + assert_eq!( + credential["identity"]["scope"]["principal_id"], + principal_id.to_string(), + "{}", + test_case.description + ); + let bearer = credential["bearer"].as_str().expect(test_case.description); + let snapshot = SnapshotInput { + location_id: Uuid::now_v7(), + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::new(), + documents: BTreeMap::new(), + }; + assert!( + matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + ), + "{}", + test_case.description + ); + let token_id = credential["identity"]["token_id"] + .as_str() + .expect(test_case.description); + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], true, "{}", test_case.description); + assert!( + matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + assert_eq!( + audit_actions(&control, bearer), + test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + #[cfg(unix)] #[test] fn given_a_committed_admin_operation_when_json_output_is_lost_then_all_response_sizes_fail_without_claiming_rollback() diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs index 62aac53bee..8999c65a6b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -5,6 +5,13 @@ pub(crate) struct AdminTestCase { pub(crate) expected_actions: [&'static str; 4], } +pub(crate) struct PrincipalAdminTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 3], +} + #[cfg(unix)] pub(crate) struct AdminDeliveryTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 4f4ffc1f3c..4da4952d73 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -35,6 +35,13 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "", expected_stderr: "Control administration failed: invalid control-plane data: location name is required\n", }, + CommandTestCase { + description: "blank principal rejected before store initialization", + arguments: &["admin", "create-principal", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: principal name is required\n", + }, CommandTestCase { description: "invalid credential ID rejected before store initialization", arguments: &["admin", "revoke-token", "not-a-uuid"], From b90a562254023f0461a3a4915fe697529a790bf2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:04:37 +0000 Subject: [PATCH 021/138] feat: expose location-scoped orchestration reads --- Cargo.lock | 2 + Cargo.toml | 2 +- .../src/api/_helpers/authentication.rs | 15 + .../src/api/_helpers/mod.rs | 2 + .../src/api/_helpers/run_reads.rs | 67 ++++ .../src/api/_helpers/snapshots.rs | 13 +- .../src/api/_helpers/trigger_reads.rs | 96 +++++ .../src/api/main/serve_api.rs | 22 +- .../sqlbuild-orchestration/src/api/models.rs | 15 +- .../src/declarations/models.rs | 14 +- .../sqlbuild-orchestration/src/runs/models.rs | 6 +- .../src/scheduler/models.rs | 11 +- crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/helpers.rs | 15 + .../tests/api/test_reads.rs | 365 ++++++++++++++++++ .../tests/api/test_types.rs | 7 + 16 files changed, 625 insertions(+), 29 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_reads.rs diff --git a/Cargo.lock b/Cargo.lock index 7f825f1ccc..c468288a19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,7 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", + "form_urlencoded", "futures-util", "http", "http-body", @@ -127,6 +128,7 @@ dependencies = [ "serde_core", "serde_json", "serde_path_to_error", + "serde_urlencoded", "sync_wrapper", "tokio", "tower", diff --git a/Cargo.toml b/Cargo.toml index ea73843c9b..b28edb4dda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ publish = false rust-version = "1.88" [workspace.dependencies] -axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio"] } +axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } fensu-policy = "=0.22.0" # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs index 2920fd6ea0..c7f6dd2e06 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -4,9 +4,11 @@ use axum::extract::{Request, State}; use axum::http::{HeaderValue, StatusCode, header}; use axum::middleware::Next; use axum::response::Response; +use uuid::Uuid; use crate::api::errors::ApiError; use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::store::models::ControlStore; pub(crate) async fn authenticate( @@ -36,3 +38,16 @@ pub(crate) async fn authenticate( .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); Ok(response) } + +pub(crate) fn require_location( + identity: &TokenIdentity, + location_id: Uuid, +) -> Result<(), ApiError> { + if identity.scope != (CredentialScope::Location { location_id }) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this location".into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 4f00fe75c5..ed6ed88be8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod authentication; pub(crate) mod client; +pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; +pub(crate) mod trigger_reads; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs new file mode 100644 index 0000000000..c8c002e3d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs @@ -0,0 +1,67 @@ +//! Location credentials can observe only their own runs; they cannot request or mutate work. + +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::ReadQuery; +use crate::credentials::models::TokenIdentity; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunNodeRecord, RunRecord}; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("limit" = Option, Query, description = "Maximum recent runs; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + list_runs(&store, (location_id, &target), options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn nodes( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + let transaction = begin(&store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", + vec![ + Parameter::Id(run_id), + Parameter::Id(location_id), + Parameter::Text(target), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "run not found".into(), + }); + } + Ok(Json(list_run_nodes(&store, run_id).await?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs index 198925e8fc..e5719f11ce 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -5,8 +5,9 @@ use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; +use crate::api::_helpers::authentication::require_location; use crate::api::errors::ApiError; -use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::credentials::models::TokenIdentity; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; @@ -53,13 +54,3 @@ pub(crate) async fn current( })?; Ok(Json(current)) } - -fn require_location(identity: &TokenIdentity, location_id: Uuid) -> Result<(), ApiError> { - if identity.scope != (CredentialScope::Location { location_id }) { - return Err(ApiError { - status: StatusCode::FORBIDDEN, - message: "credential does not authorize this location".into(), - }); - } - Ok(()) -} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs new file mode 100644 index 0000000000..ec5dfa88eb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs @@ -0,0 +1,96 @@ +//! Trigger definitions and immutable history are visible only within the authenticated location. + +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::ReadQuery; +use crate::credentials::models::TokenIdentity; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChangeRecord, TriggerRecord}; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickRecord; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(list_triggers(&store, location_id, &target).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum events; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn events( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_trigger(&store, (location_id, &target, trigger_id)).await?; + Ok(Json( + list_trigger_events(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum recent ticks; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn ticks( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_trigger(&store, (location_id, &target, trigger_id)).await?; + Ok(Json( + list_ticks(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} + +async fn require_trigger( + store: &ControlStore, + (location_id, target, trigger_id): (Uuid, &str, Uuid), +) -> Result<(), ApiError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", + vec![ + Parameter::Id(trigger_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "trigger not found".into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 2dd444bef6..b6ea25ac48 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -5,7 +5,7 @@ use axum::routing::{get, post}; use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; -use crate::api::_helpers::{runs, snapshots}; +use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; @@ -22,6 +22,26 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), ) + .route( + "/api/locations/{location_id}/targets/{target}/runs", + get(run_reads::recent), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + get(run_reads::nodes), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers", + get(trigger_reads::current), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + get(trigger_reads::events), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + get(trigger_reads::ticks), + ) .route("/api/runs/{run_id}/execution", get(runs::execution)) .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 6120833396..722d7fad9c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{runs, snapshots}; +use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( @@ -11,10 +11,21 @@ use crate::api::_helpers::{runs, snapshots}; runs::execution, runs::report, runs::heartbeat, - runs::output + runs::output, + run_reads::recent, + run_reads::nodes, + trigger_reads::current, + trigger_reads::events, + trigger_reads::ticks ))] pub(crate) struct ControlApi; +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReadQuery { + pub(crate) limit: Option, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct MutationResponse { pub applied: bool, diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 9561c50ca1..4120b8c9b6 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -16,7 +16,7 @@ pub enum OnFailure { StopGroup, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum TriggerStatus { #[default] @@ -50,7 +50,7 @@ pub struct JobDefinition { pub concurrency_groups: BTreeSet, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Schedule { Cron { @@ -63,7 +63,7 @@ pub enum Schedule { }, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct TriggerDefinition { pub description: String, @@ -83,7 +83,7 @@ pub struct Definitions { pub triggers: BTreeMap, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TriggerRecord { pub trigger_id: uuid::Uuid, pub name: String, @@ -95,7 +95,7 @@ pub struct TriggerRecord { } /// Operator changes are immutable facts; clearing an override restores the published code default. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum TriggerChange { Enable, @@ -105,7 +105,7 @@ pub enum TriggerChange { ResetCursor, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct TriggerChangeRequest { pub event_id: uuid::Uuid, @@ -115,7 +115,7 @@ pub struct TriggerChangeRequest { pub change: TriggerChange, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TriggerChangeRecord { pub request_json: TriggerChangeRequest, pub occurred_at: DateTime, diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index bf98fc240b..b6e379e372 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -48,7 +48,7 @@ impl RunOrigin { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunStatus { Queued, @@ -88,7 +88,7 @@ pub struct RunRequest { pub overlap: crate::scheduler::models::Overlap, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, pub location_id: Uuid, @@ -128,7 +128,7 @@ impl NodeOutcome { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunNodeRecord { pub node_key: String, pub attempt: i64, diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 019083278d..9f4d421736 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -47,13 +47,16 @@ pub struct CronSchedule { pub(crate) timezone: chrono_tz::Tz, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum CatchUp { #[default] Latest, /// Process the oldest bounded batch; later due times remain due for the next transaction. - All { max_runs: std::num::NonZeroUsize }, + All { + #[schema(value_type = usize, minimum = 1)] + max_runs: std::num::NonZeroUsize, + }, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -127,7 +130,7 @@ pub(crate) struct DueTrigger { pub(crate) status: TriggerStatus, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum TickOutcome { Requested, @@ -137,7 +140,7 @@ pub enum TickOutcome { Deferred, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TickRecord { pub tick_id: Uuid, pub trigger_id: Uuid, diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 543523edd6..81c5ac5beb 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -12,6 +12,8 @@ mod store_helpers; mod store_test_types; #[path = "api/test_contract.rs"] mod test_contract; +#[path = "api/test_reads.rs"] +mod test_reads; #[path = "api/test_types.rs"] mod test_types; #[path = "api/test_worker.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 4364fba3a3..bb65bdcf07 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -53,3 +53,18 @@ pub(crate) async fn stop_api(api: TestApi) { .expect("join API server") .expect("API server stopped cleanly"); } + +pub(crate) async fn read_json( + client: &reqwest::Client, + (url, bearer): (&str, &str), +) -> serde_json::Value { + let response = client + .get(url) + .bearer_auth(bearer) + .send() + .await + .expect("HTTP read"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + response.json().await.expect("typed JSON response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs new file mode 100644 index 0000000000..b2df105562 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -0,0 +1,365 @@ +//! Real HTTP reads keep both location and target scope, including guessed nested resource IDs. + +use std::collections::BTreeMap; +use std::time::Duration; + +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::declarations::models::{TriggerChange, TriggerChangeRequest}; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{read_json, start_api, stop_api}; +use crate::location_helpers::create_location; +use crate::run_helpers::{request, snapshot}; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::ApiReadTestCase; + +#[test] +fn given_published_locations_and_scheduled_work_when_read_through_http_then_resources_and_histories_remain_scoped() + { + let test_cases = [ + ApiReadTestCase { + description: "SQLite scoped HTTP reads", + backend: Backend::Sqlite, + expected_runs: 3, + expected_ticks: 3, + }, + ApiReadTestCase { + description: "PostgreSQL scoped HTTP reads", + backend: Backend::Postgres, + expected_runs: 3, + expected_ticks: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("publish declarations"); + let trigger_id = list_triggers(store, location_id, "development") + .await + .expect("trigger")[0] + .trigger_id; + let mut foreign = input.clone(); + foreign.location_id = other; + let foreign_snapshot = publish_snapshot(store, &foreign) + .await + .expect("publish other location"); + let foreign_trigger = list_triggers(store, other, "development") + .await + .expect("foreign trigger")[0] + .trigger_id; + let foreign_run = request( + (other, foreign_snapshot.snapshot_id), + 1001, + 0, + &["inventory"], + ); + request_run(store, &foreign_run) + .await + .expect("queue foreign run"); + let mut another_target = input.clone(); + another_target.target = "validation".into(); + publish_snapshot(store, &another_target) + .await + .expect("publish second target"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("evaluate due schedule"); + let mut manual_ids = Vec::new(); + for (id, node) in [(100, "customers"), (101, "products")] { + let run = request((location_id, published.snapshot_id), id, 0, &[node]); + manual_ids.push(run.run_id); + request_run(store, &run).await.expect("queue manual run"); + } + let change = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "test-admin".into(), + reason: "Pause orders refresh".into(), + change: TriggerChange::Pause, + }; + change_trigger(store, &change).await.expect("pause trigger"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let expiry = Utc::now() + chrono::Duration::hours(1); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location token"); + let principal_id = + create_principal(store, &audit, ("orders operator", PrincipalKind::Person)) + .await + .expect("unprivileged person"); + let principal = issue_token( + store, + &audit, + CredentialScope::Principal { principal_id }, + expiry, + ) + .await + .expect("principal token"); + let claim = claim_next_run(store, &lease, &BTreeMap::new()) + .await + .expect("claim") + .expect("work"); + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let prefix = format!( + "{}/api/locations/{location_id}/targets/development", + api.url + ); + let run_path = format!("{prefix}/runs"); + let node_path = format!("{prefix}/runs/{}/nodes", manual_ids[0]); + let trigger_path = format!("{prefix}/triggers"); + let event_path = format!("{trigger_path}/{trigger_id}/events"); + let tick_path = format!("{trigger_path}/{trigger_id}/ticks"); + let paths = [ + &run_path, + &node_path, + &trigger_path, + &event_path, + &tick_path, + ]; + let runs = read_json(&client, (&run_path, &token.bearer)).await; + assert_eq!( + runs.as_array().expect("runs").len(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert!( + runs.as_array() + .expect("runs") + .iter() + .all(|run| run["location_id"] == location_id.to_string() + && run["target"] == "development") + ); + let nodes = read_json(&client, (&node_path, &token.bearer)).await; + assert_eq!(nodes.as_array().expect("nodes").len(), 1); + assert_eq!(nodes[0]["node_key"], "customers"); + assert_eq!(nodes[0]["status"], "pending"); + let triggers = read_json(&client, (&trigger_path, &token.bearer)).await; + assert_eq!(triggers.as_array().expect("triggers").len(), 1); + assert_eq!(triggers[0]["trigger_id"], trigger_id.to_string()); + assert_eq!(triggers[0]["status"], "paused"); + let events = read_json(&client, (&event_path, &token.bearer)).await; + assert_eq!(events.as_array().expect("events").len(), 1); + assert_eq!(events[0]["request_json"], json!(change)); + let ticks = read_json(&client, (&tick_path, &token.bearer)).await; + assert_eq!( + ticks.as_array().expect("ticks").len(), + test_case.expected_ticks + ); + assert_eq!(ticks[0]["outcome"], "requested"); + assert_eq!(ticks[0]["scheduled_for"], "2020-01-03T00:00:00Z"); + assert_eq!(ticks[1]["outcome"], "covered"); + for path in [&run_path, &event_path, &tick_path] { + assert_eq!( + read_json(&client, (&format!("{path}?limit=1"), &token.bearer)) + .await + .as_array() + .expect("bounded history") + .len(), + 1 + ); + assert_eq!( + read_json(&client, (&format!("{path}?limit=0"), &token.bearer)).await, + json!([]) + ); + assert_eq!( + client + .get(format!("{path}?limit=invalid")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("invalid limit") + .status(), + StatusCode::BAD_REQUEST + ); + } + for path in paths { + assert_eq!( + client + .get(path) + .send() + .await + .expect("unauthenticated read") + .status(), + StatusCode::UNAUTHORIZED + ); + for bearer in [&principal.bearer, &worker.bearer] { + assert_eq!( + client + .get(path) + .bearer_auth(bearer) + .send() + .await + .expect("wrong credential scope") + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + client + .get(path.replace(&location_id.to_string(), &other.to_string())) + .bearer_auth(&token.bearer) + .send() + .await + .expect("cross-location read") + .status(), + StatusCode::FORBIDDEN + ); + } + for path in [ + node_path.replace(&manual_ids[0].to_string(), &foreign_run.run_id.to_string()), + event_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + tick_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + node_path.replace("/development/", "/validation/"), + event_path.replace("/development/", "/validation/"), + tick_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("nested ID cannot bypass scope") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + read_json( + &client, + ( + &run_path.replace("/development/", "/validation/"), + &token.bearer + ) + ) + .await, + json!([]) + ); + let validation = read_json( + &client, + ( + &trigger_path.replace("/development/", "/validation/"), + &token.bearer, + ), + ) + .await; + assert_eq!(validation.as_array().expect("other target").len(), 1); + assert_ne!(validation[0]["trigger_id"], trigger_id.to_string()); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI") + .json() + .await + .expect("OpenAPI JSON"); + for path in [ + "/api/locations/{location_id}/targets/{target}/runs", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + "/api/locations/{location_id}/targets/{target}/triggers", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + ] { + assert_eq!( + openapi["paths"][path]["get"]["security"], + json!([{"bearer": []}]) + ); + } + input.generation += 1; + input.documents.insert("triggers".into(), json!({})); + publish_snapshot(store, &input) + .await + .expect("remove declaration"); + assert_eq!( + read_json(&client, (&trigger_path, &token.bearer)).await, + json!([]) + ); + assert_eq!( + read_json(&client, (&event_path, &token.bearer)).await, + events + ); + assert_eq!(read_json(&client, (&tick_path, &token.bearer)).await, ticks); + revoke_token(store, &audit, token.identity.token_id) + .await + .expect("revoke location"); + for path in paths { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("revoked read") + .status(), + StatusCode::UNAUTHORIZED + ); + } + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 5ebd106605..2c63a4f4cd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -14,6 +14,13 @@ pub(crate) struct WorkerRedirectTestCase { pub(crate) expected_status: reqwest::StatusCode, } +pub(crate) struct ApiReadTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, + pub(crate) expected_ticks: usize, +} + pub(crate) struct TestApi { pub(crate) url: String, pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, From c836de1185d0480fdb3ebc54b8d59abdab6cdaf6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:20:30 +0000 Subject: [PATCH 022/138] feat: compile authored schedule runtime controls --- .../orchestration/edge_cases/definitions.py | 9 ++- .../compiled_project/snapshot_declarations.rs | 28 +++++-- .../orchestration_declarations/models.py | 6 +- .../compiler/compiled_project/_test_types.py | 12 ++- .../publication/test_publication.py | 29 ++++++- .../test_orchestration_snapshot.py | 80 +++++++++++++++++-- 6 files changed, 149 insertions(+), 15 deletions(-) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index db9dff756a..3fcca7b42e 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -8,7 +8,14 @@ select="tag:edge_case", settings={"run_tests": False}, concurrency=2, - schedule=schedule(cron="15 2 * * *", timezone="UTC"), + schedule=schedule( + cron="15 2 * * *", + timezone="UTC", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + catch_up="all", + max_catch_up=3, + ), ) edge_alias = edge_orders edge_factory_results = [[edge_orders]] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 4e2fce5b69..54ad885b83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -16,9 +16,9 @@ use sqlbuild_discovery::tree::main::rglob::rglob; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - JobDefinition, OnFailure, Schedule, TriggerDefinition, TriggerStatus, + JobDefinition, OnFailure, Schedule, TriggerDefinition, }; -use sqlbuild_orchestration::scheduler::models::{CatchUp, Overlap}; +use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -58,6 +58,12 @@ pub(crate) fn discover_declarations<'py>( struct AuthoredSchedule { cron: String, timezone: String, + #[pyo3(from_py_with = authored_setting)] + starts_at: Value, + default_status: String, + catch_up: String, + #[pyo3(from_py_with = authored_setting)] + max_catch_up: Value, } #[derive(FromPyObject)] @@ -128,6 +134,11 @@ pub(crate) fn compile_declarations( .map(|(kind, name)| format!("{kind}:{name}")) .collect(); if let Some(schedule) = job.schedule { + let catch_up = if schedule.max_catch_up.is_null() { + Value::String(schedule.catch_up) + } else { + serde_json::json!({schedule.catch_up: {"max_runs": schedule.max_catch_up}}) + }; triggers.insert( job.name.clone(), TriggerDefinition { @@ -137,9 +148,16 @@ pub(crate) fn compile_declarations( expression: schedule.cron, timezone: schedule.timezone, }, - starts_at: None, - default_status: TriggerStatus::Running, - catch_up: CatchUp::Latest, + starts_at: serde_json::from_value(schedule.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(schedule.default_status)) + .map_err(|error| { + PyValueError::new_err(format!("default_status: {error}")) + })?, + catch_up: serde_json::from_value(catch_up) + .map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, }, ); } diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index e3496353b1..fa8ff9efc1 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -8,10 +8,14 @@ @dataclass(frozen=True, kw_only=True) class Schedule: - """A wall-clock cron schedule attached to a job.""" + """Cron with an RFC 3339 start; catch_up='all' requires a positive max_catch_up.""" cron: str timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + catch_up: Literal["latest", "all"] = "latest" + max_catch_up: int | None = None @dataclass(frozen=True, eq=False) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 1744939e31..53f479aa3c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field +from sqlbuild.orchestration import Schedule + @dataclass(frozen=True) class HeaderProjectionTestCase: @@ -219,8 +221,15 @@ class SnapshotEnvironmentTestCase: @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str - timezone: str + schedule: Schedule expected_selection: tuple[str, ...] + expected_controls: dict[str, object] = field( + default_factory=lambda: { + "starts_at": None, + "default_status": "running", + "catch_up": "latest", + } + ) @dataclass(frozen=True) @@ -234,6 +243,7 @@ class SnapshotDeclarationRejectionTestCase: concurrency: int | None = None copies: int = 1 expected_message: str = "" + schedule: Schedule | None = None @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index ce5ca06abd..90d620f024 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -3,6 +3,8 @@ import json from pathlib import Path from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen import pytest @@ -50,7 +52,15 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical test_case: SnapshotPublicationTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "project" - write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + files: dict[str, str] = TYPE_PROOF_FILES | { + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + write_files(project_dir=project_dir, files=files) project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), @@ -71,7 +81,10 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical initial: dict[str, Any] = current_snapshot(api, test_case.target) assert frozenset(initial["nodes"]) == test_case.expected_nodes assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] - edited: dict[str, str] = TYPE_PROOF_FILES | { + assert initial["documents"]["triggers"]["edge_snapshot_orders"]["catch_up"] == { + "all": {"max_runs": 3} + } + edited: dict[str, str] = files | { "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" ) @@ -95,3 +108,15 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical current: dict[str, Any] = current_snapshot(api, test_case.target) assert current["record"] == updated assert "CAST(9 AS INTEGER)" in current["nodes"]["model:stg_orders"]["raw_sql"] + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/" + f"{quote(test_case.target, safe='')}/triggers", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + triggers: list[dict[str, Any]] = json.load(response) + assert len(triggers) == 1 + assert triggers[0]["status"] == "paused" + assert triggers[0]["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" + assert triggers[0]["definition_json"]["default_status"] == "paused" + assert triggers[0]["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 58c60b9f7a..03b75e763a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -280,11 +280,42 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang [ SnapshotScheduleTestCase( description="daylight saving zone", - timezone="Europe/London", + schedule=schedule(cron="15 2 * * *", timezone="Europe/London"), expected_selection=("model:orders",), ), SnapshotScheduleTestCase( - description="fixed offset zone", timezone="UTC", expected_selection=("model:orders",) + description="fixed offset zone", + schedule=schedule(cron="15 2 * * *", timezone="UTC"), + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="paused bounded catch-up with a zoned start", + schedule=schedule( + cron="15 2 * * *", + timezone="Europe/London", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + catch_up="all", + max_catch_up=3, + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": "2026-07-02T19:47:23.456789Z", + "default_status": "paused", + "catch_up": {"all": {"max_runs": 3}}, + }, + ), + SnapshotScheduleTestCase( + description="largest supported catch-up batch", + schedule=schedule( + cron="15 2 * * *", timezone="UTC", catch_up="all", max_catch_up=10_000 + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": None, + "default_status": "running", + "catch_up": {"all": {"max_runs": 10_000}}, + }, ), ], ids=lambda case: case.description, @@ -307,7 +338,7 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, concurrency=4, concurrency_groups=("order_reporting",), - schedule=schedule(cron="15 2 * * *", timezone=test_case.timezone), + schedule=test_case.schedule, targets=("development",), ) excluded: Job = replace(nightly, name="other_target", targets=("production",)) @@ -339,8 +370,9 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select assert trigger["schedule"] == { "kind": "cron", "expression": "15 2 * * *", - "timezone": test_case.timezone, + "timezone": test_case.schedule.timezone, } + assert {key: trigger[key] for key in test_case.expected_controls} == test_case.expected_controls edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) second: dict[str, Any] = json.loads( compile_snapshot( @@ -377,6 +409,43 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select SnapshotDeclarationRejectionTestCase( description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" ), + SnapshotDeclarationRejectionTestCase( + description="zero catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=0), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="negative catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=-1), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not a catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=True), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="unbounded all catch-up", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all"), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="catch-up exceeds transaction bound", + schedule=schedule( + cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=10_001 + ), + expected_message="catch-up exceeds 10000", + ), + SnapshotDeclarationRejectionTestCase( + description="latest does not silently ignore a bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", max_catch_up=3), + expected_message="latest.*without a bound", + ), + SnapshotDeclarationRejectionTestCase( + description="start needs an explicit offset", + schedule=schedule(cron="0 2 * * *", timezone="UTC", starts_at="2026-07-03T01:17:23"), + expected_message="starts_at", + ), SnapshotDeclarationRejectionTestCase( description="duplicate names", copies=2, expected_message="duplicate job orders" ), @@ -450,7 +519,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject select=test_case.select, settings=test_case.settings, concurrency=test_case.concurrency, - schedule=schedule(cron=test_case.cron, timezone=test_case.timezone), + schedule=test_case.schedule + or schedule(cron=test_case.cron, timezone=test_case.timezone), ) ] * test_case.copies with pytest.raises(ValueError, match=test_case.expected_message): From d3fc7fb970675c32db12d67f8f4008c01ab05d6c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:40:38 +0000 Subject: [PATCH 023/138] feat: add atomic privileged trigger controls --- .../sqlbuild-cli/src/argv/command_table.json | 90 ++++++++++++++ .../src/native_commands/_helpers/decision.rs | 32 ++++- .../src/native_commands/main/_run_admin.rs | 22 ++++ .../src/native_commands/models.rs | 7 ++ .../tests/native_commands/admin/helpers.rs | 64 ++++++++-- .../tests/native_commands/admin/test_admin.rs | 54 ++++++++- .../tests/native_commands/admin/test_types.rs | 10 +- .../tests/native_commands/test_control.rs | 112 +++++++++++++++++- .../tests/native_commands/test_types.rs | 6 + .../src/credentials/_helpers/mod.rs | 1 - .../audit.rs => main/append_audit.rs} | 2 +- .../src/credentials/main/create_location.rs | 4 +- .../src/credentials/main/create_principal.rs | 4 +- .../src/credentials/main/issue_token.rs | 4 +- .../src/credentials/main/mod.rs | 1 + .../src/credentials/main/revoke_token.rs | 4 +- .../src/declarations/main/change_trigger.rs | 32 ++++- .../tests/api/test_reads.rs | 4 +- .../tests/scheduler/ticks/helpers.rs | 48 +++++++- .../tests/scheduler/ticks/test_triggers.rs | 100 ++++++++++++---- 20 files changed, 540 insertions(+), 61 deletions(-) rename crates/sqlbuild-orchestration/src/credentials/{_helpers/audit.rs => main/append_audit.rs} (97%) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ec7cee1ac8..cd0b2193e9 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -351,6 +351,91 @@ } ] } + }, + { + "name": "change-trigger", + "parser": { + "prog": "sqb admin change-trigger", + "description": "Change runtime trigger state using privileged database access; the fact, projection and security audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TRIGGER_ID", + "required": true, + "help": "current trigger UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_trigger_change", + "metavar": "CHANGE", + "choices": [ + "enable", + "pause", + "reset-default", + "set-cursor", + "reset-cursor" + ], + "required": true, + "help": "runtime change; reset-default restores the published status default", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the operator fact and security audit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--cursor" + ], + "dest": "admin_cursor", + "help": "RFC3339 timestamp with offset; required only for set-cursor", + "group": 1 + } + ] + } } ], "choices_help": [ @@ -378,6 +463,11 @@ "dest": "revoke-token", "metavar": "revoke-token", "help": "revoke a credential by UUID" + }, + { + "dest": "change-trigger", + "metavar": "change-trigger", + "help": "change trigger status or cursor with an atomic audit and retry identity" } ] } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 4055146ffc..0673c8aa18 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -4,7 +4,9 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use chrono::{DateTime, Utc}; +use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -63,8 +65,36 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { + let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("--event-id must be a UUID".into()))?; + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if reason.trim().is_empty() { + return Err(ControlError::Invalid("--reason must not be blank".into())); + } + let mut change = json!({ + "kind": text(namespace, "admin_trigger_change").unwrap_or_default().replace('-', "_"), + }); + let cursor = text(namespace, "admin_cursor"); + if let Some(cursor) = &cursor { + change["time"] = json!(cursor); + } + let change: TriggerChange = serde_json::from_value(change) + .map_err(|error| ControlError::Invalid(format!("trigger change: {error}")))?; + if cursor.is_some() && !matches!(change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "--cursor is only valid for set-cursor".into(), + )); + } + Ok(ControlAdmin::ChangeTrigger { + event_id, + trigger_id: id, + change, + reason, + }) + } Some("issue-location-token" | "issue-principal-token") => { let expiry = DateTime::parse_from_rfc3339( &text(namespace, "admin_expires_at").unwrap_or_default(), diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 8929365faf..90bc31d618 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -6,6 +6,8 @@ use sqlbuild_orchestration::credentials::main::create_principal::create_principa use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -40,5 +42,25 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = change_trigger( + &store, + &context, + &TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason, + change, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "trigger_id": trigger_id, "applied": applied})) + } } } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index df6d08a534..b7b6f00b4b 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::models::TriggerChange; use uuid::Uuid; use crate::argv::models::Namespace; @@ -14,6 +15,12 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + ChangeTrigger { + event_id: Uuid, + trigger_id: Uuid, + change: TriggerChange, + reason: String, + }, } /// Parsing can finish an invocation before package discovery or any Python handoff. diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs index ae7e64d9c4..d43fbfa744 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -1,13 +1,29 @@ //! Run the isolated native executable; never include returned bearer bytes in assertion messages. -use std::process::Command; +use std::process::{Command, Output}; use serde_json::Value; +use sqlbuild_orchestration::declarations::models::TriggerRecord; +use uuid::Uuid; use crate::test_types::TestControl; pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { - let output = Command::new( + let output = invoke_admin(control, arguments); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration completed successfully\n" + ); + serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") +} + +pub(crate) fn invoke_admin(control: &TestControl, arguments: &[&str]) -> Output { + Command::new( control .directory .path() @@ -19,17 +35,39 @@ pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { .env("PATH", "") .env_remove("SQLBUILD_CONTROL_DATABASE_URL") .output() - .expect("native administration process"); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!( - String::from_utf8_lossy(&output.stderr), - "Opening control store...\nControl store ready\nControl administration completed successfully\n" - ); - serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") + .expect("native administration process") +} + +pub(crate) async fn read_triggers( + url: &str, + bearer: &str, + (location, target): (Uuid, &str), +) -> Vec { + let mut endpoint = reqwest::Url::parse(url).expect("API address"); + endpoint + .path_segments_mut() + .expect("API path") + .pop_if_empty() + .extend([ + "api", + "locations", + &location.to_string(), + "targets", + target, + "triggers", + ]); + reqwest::Client::new() + .get(endpoint) + .bearer_auth(bearer) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + .expect("trigger response") + .error_for_status() + .expect("authorized trigger read") + .json() + .await + .expect("trigger records") } pub(crate) fn audit_actions(control: &TestControl, bearer: &str) -> Vec { diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index 389afeb33b..f2ab692398 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -8,14 +8,14 @@ use std::collections::BTreeMap; use chrono::{Duration, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use uuid::Uuid; -use crate::admin_helpers::{administer, audit_actions}; +use crate::admin_helpers::{administer, audit_actions, invoke_admin, read_triggers}; use crate::helpers::{api_ready, start_control}; #[cfg(unix)] use crate::test_admin::test_types::AdminDeliveryTestCase; -use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase}; +use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase, TriggerAdminTestCase}; #[test] fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() @@ -29,6 +29,11 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope "location.created", "token.issued", "token.revoked", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", ], }]; let runtime = tokio::runtime::Builder::new_current_thread() @@ -78,13 +83,52 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope image_id: None, environment_hash: "orders-environment".into(), sqb_version: env!("CARGO_PKG_VERSION").into(), - nodes: BTreeMap::new(), - documents: BTreeMap::new(), + nodes: BTreeMap::from([("model:orders".into(), SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + })]), + documents: BTreeMap::from([ + ("jobs".into(), serde_json::json!({"orders": {"description": "Refresh orders.", "selection": ["model:orders"]}})), + ("triggers".into(), serde_json::json!({"daily_orders": { + "description": "Daily orders.", "job_key": "orders", "default_status": "paused", + "schedule": {"kind": "cron", "expression": "0 2 * * *", "timezone": "UTC"}, + }})), + ]), }; let published = publish_remote_snapshot(&url, bearer, &snapshot) .await .expect("operator-issued credential publishes"); assert_eq!(published.location_id, location_id); + let trigger_id = read_triggers(&url, bearer, (location_id, "development")).await[0].trigger_id.to_string(); + for case in [ + TriggerAdminTestCase { description: "enable overrides the paused code default", change: "enable", cursor_arguments: &[], expected_status: "running", expected_cursor: None }, + TriggerAdminTestCase { description: "pause overrides an enabled trigger", change: "pause", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "reset-default restores the paused code default", change: "reset-default", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "set-cursor normalizes the timestamp offset", change: "set-cursor", cursor_arguments: &["--cursor", "2026-07-03T01:17:23+05:30"], expected_status: "paused", expected_cursor: Some("2026-07-02T19:47:23+00:00") }, + TriggerAdminTestCase { description: "reset-cursor removes the durable cursor", change: "reset-cursor", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + ] { + let event_id = Uuid::now_v7().to_string(); + let mut arguments = vec!["change-trigger", &trigger_id, case.change, "--event-id", &event_id, "--reason", "Order refresh maintenance"]; + arguments.extend_from_slice(case.cursor_arguments); + for expected in [true, false] { + let result = administer(&control, &arguments); + assert_eq!(result["applied"], expected, "{}", case.description); + assert_eq!(result["event_id"], event_id); + } + let mut conflict = arguments.clone(); + conflict[6] = "Different request content"; + assert!(!invoke_admin(&control, &conflict).status.success(), "{}", case.description); + let triggers = read_triggers(&url, bearer, (location_id, "development")).await; + assert_eq!(triggers[0].status.as_str(), case.expected_status, "{}", case.description); + assert_eq!(triggers[0].last_tick_for.map(|value| value.to_rfc3339()).as_deref(), case.expected_cursor, "{}", case.description); + } + snapshot.generation = 2; + snapshot.documents.get_mut("triggers").expect("trigger document")["daily_orders"]["default_status"] = serde_json::json!("running"); + publish_remote_snapshot(&url, bearer, &snapshot).await.expect("publish a new default"); + assert_eq!(read_triggers(&url, bearer, (location_id, "development")).await[0].status.as_str(), "running", "reset-default removed the operator override"); snapshot.location_id = other_id; assert!(matches!( publish_remote_snapshot(&url, bearer, &snapshot).await, diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs index 8999c65a6b..3adc9b5e24 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -2,7 +2,7 @@ pub(crate) struct AdminTestCase { pub(crate) description: &'static str, pub(crate) location_name: &'static str, pub(crate) expected_name: &'static str, - pub(crate) expected_actions: [&'static str; 4], + pub(crate) expected_actions: [&'static str; 9], } pub(crate) struct PrincipalAdminTestCase { @@ -12,6 +12,14 @@ pub(crate) struct PrincipalAdminTestCase { pub(crate) expected_actions: [&'static str; 3], } +pub(crate) struct TriggerAdminTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) cursor_arguments: &'static [&'static str], + pub(crate) expected_status: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + #[cfg(unix)] pub(crate) struct AdminDeliveryTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 4da4952d73..74eaf195d3 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,7 +1,7 @@ use std::process::Command; use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; -use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase}; +use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase}; #[test] fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() @@ -28,6 +28,45 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin ", expected_stderr: "", }, + CommandTestCase { + description: "native trigger administration help", + arguments: &["admin", "change-trigger", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin change-trigger ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank trigger reason rejected before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "pause", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, + CommandTestCase { + description: "set-cursor requires a timestamp before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "set-cursor", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: trigger change: missing field `time`\n", + }, CommandTestCase { description: "blank location rejected before store initialization", arguments: &["admin", "create-location", " "], @@ -47,7 +86,7 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ arguments: &["admin", "revoke-token", "not-a-uuid"], expected_success: false, expected_stdout_prefix: "", - expected_stderr: "Control administration failed: invalid control-plane data: a valid location or token UUID is required\n", + expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", }, CommandTestCase { description: "invalid expiry rejected before store initialization", @@ -118,6 +157,75 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ } } +#[test] +fn given_a_cursor_free_trigger_change_when_a_cursor_is_supplied_then_it_fails_before_opening_the_store() + { + let test_cases = [ + TriggerCursorTestCase { + description: "enable rejects an extra cursor", + change: "enable", + expected_success: false, + }, + TriggerCursorTestCase { + description: "pause rejects an extra cursor", + change: "pause", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-default rejects an extra cursor", + change: "reset-default", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-cursor rejects an extra cursor", + change: "reset-cursor", + expected_success: false, + }, + ]; + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for test_case in test_cases { + for cursor in ["2026-01-01T00:00:00Z", "nonsense"] { + let output = Command::new(&binary) + .args([ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + test_case.change, + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + "--cursor", + cursor, + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert_eq!( + output.status.success(), + test_case.expected_success, + "{}: {cursor}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Control administration failed: invalid control-plane data: --cursor is only valid for set-cursor\n", + "{}: {cursor}", + test_case.description + ); + assert!( + output.stdout.is_empty(), + "{}: {cursor}", + test_case.description + ); + assert!(!directory.path().join(".sqlbuild").exists()); + } + } +} + #[test] fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_machine_readable_without_python_or_a_database() { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 15974f0788..4796bc1277 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -9,6 +9,12 @@ pub(crate) struct CommandTestCase { pub(crate) expected_stderr: &'static str, } +pub(crate) struct TriggerCursorTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) expected_success: bool, +} + pub(crate) struct OpenApiTestCase { pub(crate) description: &'static str, pub(crate) expected_path: &'static str, diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index 7093af372d..ae89a5d233 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1,2 +1 @@ -pub(crate) mod audit; pub(crate) mod scopes; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs rename to crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs index 7a78c89f96..30b2c7362d 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs @@ -8,7 +8,7 @@ use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -pub(crate) async fn audit( +pub(crate) async fn append_audit( transaction: ControlTransaction, context: &AuditContext, (event_id, action): (Uuid, &str), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs index 88d4d8d148..a046c16405 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -27,7 +27,7 @@ pub async fn create_location( "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&id, b"created"), "location.created"), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs index b85b43d51e..4b85936350 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -38,7 +38,7 @@ pub async fn create_principal( ) .await?; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&id, b"created"), "principal.created"), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs index 1ccb56aba1..090bf85c66 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -7,9 +7,9 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; use crate::credentials::constants::TOKEN_SECRET_BYTES; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -65,7 +65,7 @@ pub async fn issue_token( expires_at, }; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&token_id, b"issued"), "token.issued"), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index 49e61796cf..bde23d947a 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod append_audit; pub mod authenticate_token; pub mod create_location; pub mod create_principal; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs index 6be1bb1e78..a7366783ae 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::_helpers::audit::audit; +use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -31,7 +31,7 @@ pub async fn revoke_token( .cloned() .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; commit( - audit( + append_audit( transaction, context, (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index 4ccd6765e0..0977da705b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -1,5 +1,9 @@ //! Apply an operator fact and its projection in one transaction, with retry-safe request identity. +use serde_json::json; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; use crate::declarations::_helpers::trigger_state::trigger_timing; use crate::declarations::models::{ EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, @@ -14,11 +18,12 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; /// Authorization belongs to the API boundary; `actor` is the authenticated principal, not client input. pub async fn change_trigger( store: &ControlStore, + context: &AuditContext, request: &TriggerChangeRequest, ) -> Result { - if request.actor.trim().is_empty() { + if request.actor.trim().is_empty() || request.actor != context.actor { return Err(ControlError::Invalid( - "trigger changes require an actor".into(), + "trigger changes require the authenticated audit actor".into(), )); } let body = @@ -51,6 +56,10 @@ pub async fn change_trigger( decode_row(rows.into_iter().next().ok_or_else(|| { ControlError::Invalid("trigger does not exist in the current snapshot".into()) })?)?; + let before = json!({ + "status_override": state.status_override, + "last_tick_for": state.last_tick_for, + }); match request.change { TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), @@ -72,6 +81,23 @@ pub async fn change_trigger( "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; - commit(transaction).await?; + commit( + append_audit( + transaction, + context, + (request.event_id, "trigger.changed"), + ( + json!({"trigger_id": request.trigger_id, "snapshot_id": state.snapshot_id}), + before, + json!({ + "status_override": state.status_override, + "last_tick_for": state.last_tick_for, + "reason": request.reason, + }), + ), + ) + .await?, + ) + .await?; Ok(true) } diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index b2df105562..6849aae676 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -115,13 +115,15 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso reason: "Pause orders refresh".into(), change: TriggerChange::Pause, }; - change_trigger(store, &change).await.expect("pause trigger"); let audit = AuditContext { actor: "test-admin".into(), granting_role: "admin".into(), request_id: Uuid::now_v7(), ip_address: None, }; + change_trigger(store, &audit, &change) + .await + .expect("pause trigger"); let expiry = Utc::now() + chrono::Duration::hours(1); let token = issue_token( store, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 1cbed7a105..7d4bac4e82 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; -use serde_json::json; +use serde_json::{Value, json}; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; use sqlbuild_orchestration::runs::main::list_runs::list_runs; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; @@ -14,6 +14,7 @@ use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::fault_helpers::{install_fault, remove_fault}; @@ -59,6 +60,51 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { } } +async fn trigger_audit_rows(store: &ControlStore) -> Vec<(String, String, String, String, String)> { + let sql = "SELECT CAST(event_id AS TEXT), actor, granting_role, CAST(before_json AS TEXT), CAST(after_json AS TEXT) FROM sqb_security_audit WHERE action='trigger.changed'"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect trigger audit") + .prepare(sql) + .expect("audit query") + .query_map([], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }) + .expect("read trigger audits") + .collect::>() + .expect("audit rows"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_all(pool) + .await + .expect("trigger audits") +} + +pub(crate) async fn trigger_audits(store: &ControlStore) -> Vec { + trigger_audit_rows(store) + .await + .into_iter() + .map(|(event_id, actor, granting_role, before, after)| { + json!({ + "event_id": event_id, + "actor": actor, + "granting_role": granting_role, + "before_json": serde_json::from_str::(&before).expect("before state"), + "after_json": serde_json::from_str::(&after).expect("after state"), + }) + }) + .collect() +} + pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { let fixture = open_test_store(backend).await?; let store = &fixture.store; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index cf2cfb68ac..34e9c4b198 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -1,5 +1,6 @@ //! Operator trigger facts remain consistent across publication and retry boundaries. +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; @@ -16,7 +17,7 @@ use crate::fault_helpers::{install_fault, remove_fault}; use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::ticks::helpers::snapshot; +use crate::ticks::helpers::{snapshot, trigger_audits}; use crate::ticks::test_types::{TransactionFault, TriggerControlsTestCase}; #[test] @@ -59,7 +60,13 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ reason: "maintenance".into(), change: TriggerChange::Pause, }; - assert!(change_trigger(store, &pause).await.expect("pause")); + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!(change_trigger(store, &audit, &pause).await.expect("pause")); let paused = list_triggers(store, location, "development") .await .expect("paused trigger") @@ -86,27 +93,49 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::Enable, ..pause.clone() }; - let fault = TransactionFault { - description: "operator event and projection", - table: "sqb_trigger_state", - operation: "UPDATE", - condition: "", - sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", - postgres_body: "RAISE EXCEPTION 'injected state failure'", - }; - install_fault(store, &fault).await; - assert!(change_trigger(store, &enable).await.is_err()); - remove_fault(store, &fault).await; - assert_eq!( - list_trigger_events(store, trigger_id, 100) + for fault in [ + TransactionFault { + description: "operator event and projection", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", + postgres_body: "RAISE EXCEPTION 'injected state failure'", + }, + TransactionFault { + description: "operator security audit", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }, + ] { + install_fault(store, &fault).await; + assert!(change_trigger(store, &audit, &enable).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_trigger_events(store, trigger_id, 100) + .await + .expect("rolled-back fact") + .len(), + 1 + ); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("rolled back projection")[0] + .status, + TriggerStatus::Paused + ); + } + assert!( + change_trigger(store, &audit, &enable) .await - .expect("rolled-back fact") - .len(), - 1 + .expect("retry enable") ); - assert!(change_trigger(store, &enable).await.expect("retry enable")); assert!( - !change_trigger(store, &pause) + !change_trigger(store, &audit, &pause) .await .expect("retry old pause is inert") ); @@ -115,7 +144,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ ..pause.clone() }; assert!( - change_trigger(store, &conflict).await.is_err(), + change_trigger(store, &audit, &conflict).await.is_err(), "same event ID cannot change content" ); input.generation = 2; @@ -139,7 +168,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::ResetDefault, ..pause.clone() }; - change_trigger(store, &reset) + change_trigger(store, &audit, &reset) .await .expect("restore new code default"); assert_eq!( @@ -157,7 +186,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::SetCursor { time }, ..pause.clone() }; - change_trigger(store, &cursor) + change_trigger(store, &audit, &cursor) .await .expect("set durable cursor"); assert_eq!( @@ -172,7 +201,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ change: TriggerChange::ResetCursor, ..pause.clone() }; - change_trigger(store, &reset_cursor) + change_trigger(store, &audit, &reset_cursor) .await .expect("reset cursor"); assert!( @@ -191,6 +220,29 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ .iter() .all(|event| event.request_json.actor == "orders-operator") ); + let other_actor = AuditContext { + actor: "customers-operator".into(), + ..audit.clone() + }; + assert!(change_trigger(store, &other_actor, &pause).await.is_err()); + let audits = trigger_audits(store).await; + assert_eq!(audits.len(), test_case.expected_events); + assert!( + audits + .iter() + .all(|row| row["actor"] == "orders-operator" + && row["granting_role"] == "operator") + ); + let cleared = audits + .iter() + .find(|row| row["event_id"] == reset_cursor.event_id.to_string()) + .expect("cursor reset audit"); + assert_eq!( + cleared["before_json"]["last_tick_for"], + "2020-01-02T00:00:00Z" + ); + assert!(cleared["after_json"]["last_tick_for"].is_null()); + assert_eq!(cleared["after_json"]["reason"], "maintenance"); close_test_store(fixture).await; }); } From 9301ea035078e9994914e083f6a05fa5578ec556 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:08:11 +0000 Subject: [PATCH 024/138] feat: deliver claim-scoped Recall through native workers --- .../src/api/_helpers/runs.rs | 23 +- .../src/api/constants.rs | 3 + .../src/api/main/serve_api.rs | 4 +- crates/sqlbuild-orchestration/src/api/mod.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/runs/_helpers/mod.rs | 1 + .../src/runs/_helpers/output.rs | 8 +- .../src/runs/_helpers/recall.rs | 103 +++++ .../src/runs/constants.rs | 2 + .../src/runs/main/mod.rs | 1 + .../src/runs/main/publish_run_output.rs | 4 +- .../src/runs/main/publish_run_recall.rs | 39 ++ .../sqlbuild-orchestration/src/runs/models.rs | 17 +- .../src/worker/_helpers/mod.rs | 1 + .../src/worker/_helpers/recall.rs | 215 +++++++++ crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/output/helpers.rs | 2 +- .../tests/api/recall/helpers.rs | 411 ++++++++++++++++++ .../tests/api/test_contract.rs | 30 +- .../tests/api/test_types.rs | 7 + .../tests/api/test_worker.rs | 102 ++++- .../src/records/_helpers/mod.rs | 1 + .../src/records/_helpers/record_row.rs | 61 +++ crates/sqlbuild-recall/src/records/models.rs | 12 +- crates/sqlbuild-recall/src/schema/models.rs | 15 +- crates/sqlbuild-recall/src/store/models.rs | 6 +- 26 files changed, 1046 insertions(+), 26 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs create mode 100644 crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/recall/helpers.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/record_row.rs diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index a0f39f3324..3762a428bf 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -4,6 +4,7 @@ use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::{Extension, Json}; use sqlbuild_recall::records::models::OutputChunk; +use sqlbuild_recall::store::models::InsertOutcome; use uuid::Uuid; use crate::api::errors::ApiError; @@ -12,8 +13,11 @@ use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::runs::main::change_run::change_run; use crate::runs::main::heartbeat_run::heartbeat_run; use crate::runs::main::publish_run_output::publish_run_output; +use crate::runs::main::publish_run_recall::publish_run_recall; use crate::runs::main::run_execution::run_execution; -use crate::runs::models::{OutputPublication, RunAuthority, RunChangeRequest, RunExecution}; +use crate::runs::models::{ + OutputPublication, RecallBatch, RunAuthority, RunChangeRequest, RunExecution, +}; use crate::store::models::ControlStore; #[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), @@ -87,6 +91,23 @@ pub(crate) async fn output( )) } +#[utoipa::path(post, path = "/api/runs/{run_id}/recall", params(("run_id" = Uuid, Path)), + request_body = RecallBatch, + responses((status = 200, body = InsertOutcome), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(batch): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_recall(&store, (run_id, claim_id), &batch).await?, + )) +} + fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { if let CredentialScope::Run { run_id: permitted, diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs new file mode 100644 index 0000000000..d3d0c93f8d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -0,0 +1,3 @@ +//! Shared HTTP transport bounds for the server and native clients. + +pub(crate) const BODY_BYTES: usize = 64 * 1024 * 1024; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index b6ea25ac48..52c2efba75 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -6,6 +6,7 @@ use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; +use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; @@ -46,7 +47,8 @@ pub async fn serve_api( .route("/api/runs/{run_id}/events", post(runs::report)) .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) .route("/api/runs/{run_id}/output", post(runs::output)) - .layer(DefaultBodyLimit::max(64 * 1024 * 1024)) + .route("/api/runs/{run_id}/recall", post(runs::recall)) + .layer(DefaultBodyLimit::max(BODY_BYTES)) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); let application = protected .route( diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index 8fa5a583a4..951756c642 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -1,6 +1,7 @@ //! Python-free HTTP endpoints; locations and workers have no direct database access. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 722d7fad9c..c2604cc5c5 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -12,6 +12,7 @@ use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; runs::report, runs::heartbeat, runs::output, + runs::recall, run_reads::recent, run_reads::nodes, trigger_reads::current, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index 5bfdb31c61..a3adbd20a7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -5,3 +5,4 @@ pub(crate) mod nodes; pub(crate) mod output; pub(crate) mod projection; pub(crate) mod publish_change; +pub(crate) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs index d6daa4602a..beb7a601ef 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs @@ -6,13 +6,13 @@ use sqlbuild_recall::records::models::{OutputChunk, RecallRow, RowEnvelope}; use uuid::Uuid; use crate::runs::constants::{OUTPUT_BATCH_BYTES, OUTPUT_BATCH_CHUNKS}; -use crate::runs::models::OutputMetadata; +use crate::runs::models::ClaimMetadata; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -pub(crate) async fn output_envelope( +pub(crate) async fn claim_envelope( transaction: ControlTransaction, (run_id, claim_id): (Uuid, Uuid), ) -> Result<(ControlTransaction, RowEnvelope), ControlError> { @@ -24,8 +24,8 @@ pub(crate) async fn output_envelope( vec![Parameter::Id(run_id), Parameter::Id(claim_id)], ) .await?; - let metadata: OutputMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { - ControlError::Invalid("output does not belong to an active run claim".into()) + let metadata: ClaimMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("publication does not belong to an active run claim".into()) })?)?; Ok(( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs new file mode 100644 index 0000000000..442b0ba521 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs @@ -0,0 +1,103 @@ +//! Global Recall keys cannot escape a worker's claim namespace. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{RecallRow, RecallValue, RowEnvelope}; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::models::InsertOutcome; + +use crate::runs::models::RecallRecord; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Recall statement projections join correlations globally, not only by invocation. +pub(crate) fn claim_record( + envelope: &RowEnvelope, + record: &RecallRecord, +) -> Result { + let mut values = record.values.clone(); + for name in [ + "event_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", + ] { + if let Some(RecallValue::Text(id)) = values.get_mut(name) { + *id = serde_json::json!([envelope.invocation_id, id]).to_string(); + } + } + match record.table { + Table::Meta | Table::Conflicts => { + return Err(ControlError::Invalid( + "Recall store metadata is server-owned".into(), + )); + } + Table::Artifacts => { + let Some(RecallValue::Blob(content)) = values.get("content") else { + return Err(ControlError::Invalid( + "Recall artifact content must be bytes".into(), + )); + }; + if values.get("digest") + != Some(&RecallValue::Text(format!( + "sha256:{}", + sha256_hex(content) + ))) + || values.get("size") != Some(&RecallValue::Integer(content.len() as i64)) + { + return Err(ControlError::Invalid( + "Recall artifact digest or size differs from its bytes".into(), + )); + } + } + _ => {} + } + RecallRow::from_values(envelope, record.table, &values) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) async fn append_records( + mut transaction: ControlTransaction, + envelope: &RowEnvelope, + rows: &[RecallRow], +) -> Result<(ControlTransaction, InsertOutcome), ControlError> { + let mut result = InsertOutcome::default(); + let mut delivered: BTreeMap = BTreeMap::new(); + for row in rows { + if row.table == Table::ArtifactRefs + && !rows.iter().any(|candidate| { + candidate.table == Table::Artifacts + && candidate.value("digest") == row.value("digest") + }) + { + let Some(RecallValue::Text(digest)) = row.value("digest") else { + return Err(ControlError::Invalid( + "Recall artifact digest must be text".into(), + )); + }; + let references; + (transaction, references) = query(transaction, + "SELECT 1 AS present FROM recall_artifact_refs WHERE invocation_id=$1 AND digest=$2 LIMIT 1", + vec![Parameter::Text(envelope.invocation_id.clone()), Parameter::Text(digest.clone())]).await?; + if references.is_empty() { + return Err(ControlError::Invalid( + "Recall artifact reference has no invocation-owned content".into(), + )); + } + } + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + if let Some(conflict) = outcome.conflict { + result.conflicts.push(conflict); + } else { + *delivered.entry(row.table).or_insert(0) += 1; + } + } + result.delivered = delivered.into_iter().collect(); + Ok((transaction, result)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index f53d82c1b5..661f08ec4f 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -8,3 +8,5 @@ pub(crate) const MONITOR_BATCH_SIZE: usize = 100; pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; +pub(crate) const RECALL_BATCH_ROWS: usize = 500; +pub(crate) const RECALL_INVOCATION_ID_BYTES: usize = 256; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 46615363df..da3eca538f 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -5,6 +5,7 @@ pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; pub mod publish_run_output; +pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs index 35633a3a0e..8390019aab 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -3,7 +3,7 @@ use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; -use crate::runs::_helpers::output::{output_envelope, output_rows}; +use crate::runs::_helpers::output::{claim_envelope, output_rows}; use crate::runs::models::OutputPublication; use crate::store::errors::ControlError; use crate::store::main::append_recall::append_recall; @@ -18,7 +18,7 @@ pub async fn publish_run_output( chunks: &[OutputChunk], ) -> Result { let transaction = begin(store, TransactionMode::Write).await?; - let (mut transaction, envelope) = output_envelope(transaction, (run_id, claim_id)).await?; + let (mut transaction, envelope) = claim_envelope(transaction, (run_id, claim_id)).await?; let mut published = OutputPublication::default(); for row in output_rows(&envelope, chunks)? { let outcome; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs new file mode 100644 index 0000000000..9a70513ebd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -0,0 +1,39 @@ +//! A worker's complete Recall batch and active-claim check share one transaction. + +use sqlbuild_recall::store::models::InsertOutcome; +use uuid::Uuid; + +use crate::runs::_helpers::output::claim_envelope; +use crate::runs::_helpers::recall::{append_records, claim_record}; +use crate::runs::constants::{RECALL_BATCH_ROWS, RECALL_INVOCATION_ID_BYTES}; +use crate::runs::models::RecallBatch; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn publish_run_recall( + store: &ControlStore, + identity: (Uuid, Uuid), + batch: &RecallBatch, +) -> Result { + if batch.invocation_id.trim().is_empty() + || batch.invocation_id.len() > RECALL_INVOCATION_ID_BYTES + || batch.records.len() > RECALL_BATCH_ROWS + { + return Err(ControlError::Invalid( + "invalid Recall invocation or batch size".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, mut envelope) = claim_envelope(transaction, identity).await?; + envelope.invocation_id = format!("{}/{}", envelope.invocation_id, batch.invocation_id); + let rows = batch + .records + .iter() + .map(|record| claim_record(&envelope, record)) + .collect::, _>>()?; + let (transaction, result) = append_records(transaction, &envelope, &rows).await?; + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index b6e379e372..c0496bd605 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -273,8 +273,23 @@ pub struct OutputPublication { pub conflicts: u32, } +/// One invocation's records; the server owns its claim namespace, start time and version. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallBatch { + pub invocation_id: String, + pub records: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallRecord { + pub table: sqlbuild_recall::schema::models::Table, + pub values: BTreeMap, +} + #[derive(Debug, Deserialize)] -pub(crate) struct OutputMetadata { +pub(crate) struct ClaimMetadata { pub(crate) claimed_at: DateTime, pub(crate) sqb_version: String, } diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs index 9ec828d671..e3fde3c92a 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod client; pub(crate) mod heartbeat; +pub(crate) mod recall; pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs new file mode 100644 index 0000000000..b259195962 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs @@ -0,0 +1,215 @@ +//! Reuse the native delivery thread and accounting over claim-scoped HTTP, without Python. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::models::InsertOutcome; +use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; +use sqlbuild_recall::writer::constants::{BATCH_ROWS, BATCH_WINDOW_MS}; +use sqlbuild_recall::writer::models::SessionStore; + +use crate::api::constants::BODY_BYTES; +use crate::api::main::client_response::client_response; +use crate::runs::constants::RECALL_BATCH_ROWS; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::worker::models::WorkerClient; + +#[derive(Debug)] +struct WorkerRecallOpener(WorkerClient); + +struct WorkerRecallStore { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + invocation_id: String, +} + +fn failure(message: impl Into, retryable: bool) -> StoreError { + StoreError { + kind: if retryable { + StoreErrorKind::Busy + } else { + StoreErrorKind::Other + }, + message: message.into(), + } +} + +fn transport_record(row: &RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + let value = row.value(column.name)?; + Some((column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +impl WorkerClient { + /// Pass this to the native Recall session; all delivery finishes before terminal reporting. + #[must_use] + pub fn recall_store(&self) -> SessionStore { + SessionStore::Hosted(Arc::new(WorkerRecallOpener(self.clone()))) + } + + /// The server owns common envelope columns and recomputes each row's content hash. + pub async fn publish_recall( + &self, + batch: &RecallBatch, + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}recall", self.run_url)) + .json(batch), + ) + .await?) + } +} + +impl RecallStoreOpener for WorkerRecallOpener { + fn open( + &self, + envelope: &RowEnvelope, + give_up: Instant, + ) -> Result, StoreError> { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| failure("Recall delivery runtime could not start", false))?; + let client = { + let _entered = runtime.enter(); + WorkerClient { + connection: self + .0 + .connection + .independent() + .map_err(|_| failure("Recall HTTP client could not start", false))?, + ..self.0.clone() + } + }; + Ok(Box::new(WorkerRecallStore { + runtime, + client, + invocation_id: envelope.invocation_id.clone(), + })) + } + + fn batch_rows(&self) -> usize { + BATCH_ROWS + } + + fn batch_window(&self) -> Duration { + Duration::from_millis(BATCH_WINDOW_MS) + } +} + +/// Native materialization emits content/reference pairs, whose proof stays in one HTTP request. +impl RecallStore for WorkerRecallStore { + fn insert(&self, rows: &[RecallRow], give_up: Instant) -> Result { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let mut batch = RecallBatch { + invocation_id: self.invocation_id.clone(), + records: Vec::new(), + }; + let base_bytes = serde_json::to_vec(&batch) + .map_err(|_| failure("Recall batch could not be encoded", false))? + .len(); + self.runtime.block_on(async { + let mut result = InsertOutcome::default(); + let mut bytes = base_bytes; + let mut offset = 0; + while offset < rows.len() { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let paired = rows[offset].table == Table::Artifacts + && rows.get(offset + 1).is_some_and(|next| { + next.table == Table::ArtifactRefs + && next.value("digest") == rows[offset].value("digest") + }); + let end = offset + if paired { 2 } else { 1 }; + let records = rows[offset..end] + .iter() + .map(transport_record) + .collect::>(); + let added = serde_json::to_vec(&records) + .map_err(|_| failure("Recall records could not be encoded", false))? + .len() + - 2; + if base_bytes + added > BODY_BYTES { + return Err(failure("Recall record exceeds the HTTP body limit", false)); + } + if !batch.records.is_empty() + && (batch.records.len() + records.len() > RECALL_BATCH_ROWS + || bytes + added + 1 > BODY_BYTES) + { + (batch, result) = self.flush(batch, result, give_up).await?; + bytes = base_bytes; + } + bytes += added + usize::from(!batch.records.is_empty()); + batch.records.extend(records); + offset = end; + } + if !batch.records.is_empty() || rows.is_empty() { + (_, result) = self.flush(batch, result, give_up).await?; + } + result.delivered.sort_by_key(|(table, _)| *table); + Ok(result) + }) + } +} + +impl WorkerRecallStore { + async fn flush( + &self, + mut batch: RecallBatch, + mut result: InsertOutcome, + give_up: Instant, + ) -> Result<(RecallBatch, InsertOutcome), StoreError> { + let outcome = tokio::time::timeout_at( + tokio::time::Instant::from_std(give_up), + self.client.publish_recall(&batch), + ) + .await + .map_err(|_| failure("Recall delivery deadline expired", true))? + .map_err(|error| { + let retryable = match &error { + crate::worker::errors::WorkerError::Transport(_) => true, + crate::worker::errors::WorkerError::Rejected(status) => { + status.is_server_error() || *status == reqwest::StatusCode::TOO_MANY_REQUESTS + } + _ => false, + }; + failure(error.to_string(), retryable) + })?; + for (table, count) in outcome.delivered { + if let Some((_, previous)) = result + .delivered + .iter_mut() + .find(|(found, _)| *found == table) + { + *previous += count; + } else { + result.delivered.push((table, count)); + } + } + result.conflicts.extend(outcome.conflicts); + result.failed.extend(outcome.failed); + result.failure = result.failure.take().or(outcome.failure); + batch.records.clear(); + Ok((batch, result)) + } +} diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 81c5ac5beb..f72b32d6fb 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -4,6 +4,8 @@ mod helpers; mod location_helpers; #[path = "api/output/helpers.rs"] mod output_helpers; +#[path = "api/recall/helpers.rs"] +mod recall_helpers; #[path = "runs/helpers.rs"] mod run_helpers; #[path = "store/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs index 6c628f05de..230312a314 100644 --- a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -21,7 +21,7 @@ pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputC } } -async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { +pub(crate) async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; let ControlStore::Postgres(pool) = store else { let ControlStore::Sqlite(path) = store else { diff --git a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs new file mode 100644 index 0000000000..edc555b357 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs @@ -0,0 +1,411 @@ +//! Exercise native writer delivery and authenticated retries against both physical stores. + +use std::time::Duration; + +use serde_json::json; +use sqlbuild_orchestration::runs::models::{RecallBatch, RecallRecord}; +use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::models::WorkerClient; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{ + ArtifactInput, InvocationContext, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; +use sqlbuild_recall::writer::models::SessionConfig; +use uuid::Uuid; + +use crate::output_helpers::stored_bytes; + +fn record(row: RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + row.value(column.name) + .map(|value| (column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +pub(crate) async fn recall_contract( + store: &ControlStore, + worker: &WorkerClient, + claim_id: Uuid, +) -> RecallBatch { + let envelope = RowEnvelope { + invocation_id: "execution".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "untrusted-version".into(), + }; + let invocation = format!("worker/{claim_id}/execution"); + let event = json!({ + "event_id": "same-event-id", "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "schema_version": 1, "producer": "executor", + "statement_id": "shared-producer-id", + "payload": {"duration_ms": 12.75, "row_count": 9, "outcome": "success"} + }); + let mut batch = RecallBatch { + invocation_id: envelope.invocation_id.clone(), + records: vec![record( + event_object_row(&envelope, event.as_object().expect("event object")) + .expect("encode event"), + )], + }; + for _ in 0..2 { + let outcome = worker + .publish_recall(&batch) + .await + .expect("retry event batch"); + assert_eq!(outcome.delivered, [(Table::Events, 1)]); + assert!(outcome.conflicts.is_empty()); + } + let events = read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read stored events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["duration_ms"], json!(12.75)); + assert_eq!(events[0]["row_count"], json!(9)); + assert_ne!(events[0]["event_id"], json!("same-event-id")); + assert_ne!(events[0]["producer_version"], json!("untrusted-version")); + batch.records[0] + .values + .insert("duration_ms".into(), RecallValue::Real(13.25)); + assert_eq!( + worker + .publish_recall(&batch) + .await + .expect("publish conflicting event") + .conflicts + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read original event")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("read conflict evidence") + .len(), + 1 + ); + + let mut separate = batch.clone(); + separate.invocation_id = "another-invocation".into(); + assert_eq!( + worker + .publish_recall(&separate) + .await + .expect("publish another invocation") + .delivered, + [(Table::Events, 1)] + ); + assert_eq!( + read_recall_view(store, ("statements", &invocation), 10) + .await + .expect("read first invocation statement")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view( + store, + ( + "statements", + &format!("worker/{claim_id}/another-invocation") + ), + 10 + ) + .await + .expect("read second invocation statement")[0]["duration_ms"], + json!(13.25) + ); + let artifact = ArtifactInput { + digest: format!("sha256:{}", sha256_hex(b"orders\0\xff")), + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + content: b"orders\0\xff".to_vec(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: "first".into(), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }; + let [content, reference] = artifact_rows(&envelope, &artifact).map(record); + let artifacts = RecallBatch { + invocation_id: "execution".into(), + records: vec![content.clone(), reference.clone()], + }; + assert_eq!( + worker + .publish_recall(&artifacts) + .await + .expect("publish artifact with proof") + .delivered + .len(), + 2 + ); + let mut later_ref = reference.clone(); + later_ref + .values + .insert("ref_id".into(), RecallValue::Text("second".into())); + assert_eq!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records: vec![later_ref] + }) + .await + .expect("publish later artifact reference") + .delivered, + [(Table::ArtifactRefs, 1)] + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read both artifact references") + .len(), + 2 + ); + + let mut forged = content; + forged + .values + .insert("content".into(), RecallValue::Blob(b"different".to_vec())); + let mut event_next = batch.records[0].clone(); + event_next.values.insert( + "event_id".into(), + RecallValue::Text("must-roll-back".into()), + ); + let mut forged_envelope = event_next.clone(); + forged_envelope.values.insert( + "invocation_id".into(), + RecallValue::Text("scheduler/other".into()), + ); + let mut wrong_type = event_next.clone(); + wrong_type + .values + .insert("row_count".into(), RecallValue::Text("nine".into())); + let mut missing = event_next.clone(); + missing.values.remove("event_id"); + let mut bad_json = event_next.clone(); + bad_json + .values + .insert("payload".into(), RecallValue::Text("not-json".into())); + let mut bad_timestamp = event_next.clone(); + bad_timestamp + .values + .insert("occurred_at".into(), RecallValue::Text("not-a-time".into())); + for records in [ + vec![event_next.clone(), forged], + vec![event_next.clone(), forged_envelope], + vec![event_next.clone(), wrong_type], + vec![event_next.clone(), missing], + vec![event_next.clone(), bad_json], + vec![event_next.clone(), bad_timestamp], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Meta, + values: Default::default(), + }, + ], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Conflicts, + values: Default::default(), + }, + ], + vec![event_next; 501], + ] { + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + // A later authorization failure rolls back the preceding valid event, not just validation errors. + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "foreign".into(), + records: vec![batch.records[0].clone(), reference] + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + assert!( + read_recall_view(store, ("events", &format!("worker/{claim_id}/foreign")), 10) + .await + .expect("read rolled-back invocation") + .is_empty() + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read unchanged invocation") + .len(), + 1 + ); + + native_session(store, worker, claim_id).await; + large_batches(store, worker, claim_id).await; + batch +} + +async fn native_session(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let hosted = worker.recall_store(); + let summary = tokio::task::spawn_blocking(move || { + let session = start_recall_session(SessionConfig { + store: hosted, + envelope: RowEnvelope { invocation_id: "native-writer".into(), invocation_started_at: "2026-01-02T03:04:05Z".into(), producer_version: "test-version".into() }, + context: InvocationContext { command: Some("build".into()), git_dirty: Some(true), ..Default::default() }, + retention_days: 30, shutdown_timeout: Duration::from_secs(10), diagnostics_min_severity: "debug".into(), + max_output_bytes: None, heartbeat_interval: Duration::from_secs(60), setup_budget: Duration::from_secs(2), + maintenance: false, defer_context: false, maintenance_owner: "test-worker".into(), audit_evidence: Default::default(), + }); + session.append_output(OutputStream::Stdout, b"customers\0\xff\n"); + session.append_output(OutputStream::Stderr, b"warning\xc3\x28\n"); + session.publish_sql("select 7 as order_id", Some("statement-1"), Some("orders")); + session.publish_diagnostic(&json!({"occurred_at": "2026-01-02T03:04:05Z", "severity": "warning", "logger": "sqlbuild", "source": "test", "message": "orders warning", "fields": {"count": 3}}).to_string()).expect("queue diagnostic"); + session.publish_event(&json!({"event_id": "native-event", "occurred_at": "2026-01-02T03:04:05Z", "event_type": "statement_completed", "producer": "executor", "schema_version": 1, "payload": {"duration_ms": 2.5}}).to_string()).expect("queue event"); + session.close() + }).await.expect("native writer thread"); + assert!(summary.flush_complete, "{summary:?}"); + assert_eq!(summary.total().accepted, summary.total().delivered); + let invocation = format!("worker/{claim_id}/native-writer"); + let output = stored_bytes(store, &invocation).await; + assert_eq!(output.len(), 2); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"customers\0\xff\n") + ); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"warning\xc3\x28\n") + ); + assert_eq!( + read_recall_view(store, ("diagnostics", &invocation), 10) + .await + .expect("read diagnostic")[0]["message"], + json!("orders warning") + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read native events") + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read captured SQL")[0]["text"], + json!("select 7 as order_id") + ); + let delivered = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("read delivery accounting"); + assert_eq!(delivered.len(), 1); + assert!( + delivered[0]["flush_complete"] == json!(1) || delivered[0]["flush_complete"] == json!(true) + ); +} + +async fn large_batches(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + use sqlbuild_recall::writer::models::SessionStore; + use std::time::Instant; + + let SessionStore::Hosted(opener) = worker.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "backlog".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(2)) + .expect("open native HTTP store"); + // A replacement can expand one pending lifecycle event beyond the queue's batch count. + let events = (0..501) + .map(|index| { + event_object_row(&envelope, json!({ + "event_id": format!("backlog-{index}"), "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "producer": "executor", "schema_version": 1, + "payload": {"row_count": index} + }).as_object().expect("event object")).expect("encode backlog event") + }) + .collect::>(); + let expanded = store.insert(&events, Instant::now() + Duration::from_secs(30)); + // Six maximum-sized native artifacts fit the 32 MiB queue, but their JSON blobs exceed 64 MiB. + let artifacts = (0..6) + .flat_map(|index| { + let content = vec![b'a' + index; 4 * 1024 * 1024]; + artifact_rows( + &envelope, + &ArtifactInput { + digest: format!("sha256:{}", sha256_hex(&content)), + content, + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: format!("backlog-{index}"), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }, + ) + }) + .collect::>(); + let bytes = store.insert(&artifacts, Instant::now() + Duration::from_secs(30)); + assert!( + expanded.is_ok() && bytes.is_ok(), + "materialized rows: {expanded:?}; wire bytes: {bytes:?}" + ); + assert_eq!( + expanded.expect("publish expanded event batch").delivered, + [(Table::Events, 501)] + ); + assert_eq!( + bytes.expect("publish large artifact batch").delivered, + [(Table::Artifacts, 6), (Table::ArtifactRefs, 6)] + ); + }) + .await + .expect("join backlog writer"); + let invocation = format!("worker/{claim_id}/backlog"); + assert_eq!( + read_recall_view(store, ("events", &invocation), 1000) + .await + .expect("read full event backlog") + .len(), + 501 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read large artifacts") + .len(), + 6 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 72ea9f093b..df890b2024 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -378,17 +378,25 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie (Uuid::now_v7(), &worker.bearer), (run_id, &location_token.bearer), ] { - assert_eq!( - client - .post(format!("{}/api/runs/{output_run}/output", api.url)) - .bearer_auth(credential) - .json(&json!([])) - .send() - .await - .expect("output scope denial") - .status(), - StatusCode::FORBIDDEN - ); + for (endpoint, body) in [ + ("output", json!([])), + ( + "recall", + json!({"invocation_id": "execution", "records": []}), + ), + ] { + assert_eq!( + client + .post(format!("{}/api/runs/{output_run}/{endpoint}", api.url)) + .bearer_auth(credential) + .json(&body) + .send() + .await + .expect("publication scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } } let mut forged = json!(crate::output_helpers::chunk( 0, diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 2c63a4f4cd..550618fdb8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -14,6 +14,13 @@ pub(crate) struct WorkerRedirectTestCase { pub(crate) expected_status: reqwest::StatusCode, } +pub(crate) struct WorkerDeadlineTestCase { + pub(crate) description: &'static str, + pub(crate) budget: std::time::Duration, + pub(crate) expected_kind: sqlbuild_recall::store::errors::StoreErrorKind, + pub(crate) expected_arrival: bool, +} + pub(crate) struct ApiReadTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index e488f61f11..07a1beb282 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -24,10 +24,13 @@ use uuid::Uuid; use crate::helpers::{start_api, stop_api}; use crate::location_helpers::create_location; use crate::output_helpers::{chunk, output_contract}; +use crate::recall_helpers::recall_contract; use crate::run_helpers::{queue, request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{ApiContractTestCase, TestApi, WorkerRedirectTestCase}; +use crate::test_types::{ + ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, +}; #[test] fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() @@ -142,6 +145,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a .await .expect("verified start"); output_contract(store, &worker, identity.1).await; + let recall_batch = recall_contract(store, &worker, identity.1).await; assert_eq!(session.execution.claim, claim, "{}", test_case.description); assert_eq!( start_worker(&worker, &baseline) @@ -208,6 +212,10 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a ) .await .expect("finish"); + assert!(matches!( + worker.publish_recall(&recall_batch).await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); assert!(matches!( worker .publish_output(&[chunk( @@ -370,3 +378,95 @@ fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_no }); } } + +#[test] +fn given_a_slow_recall_server_when_the_delivery_deadline_expires_then_the_native_store_stops_waiting() + { + use sqlbuild_recall::records::models::RowEnvelope; + use sqlbuild_recall::store::errors::StoreErrorKind; + use sqlbuild_recall::writer::models::SessionStore; + use std::time::{Duration, Instant}; + + let test_cases = [ + WorkerDeadlineTestCase { + description: "HTTP wait expires within the close budget", + budget: Duration::from_millis(50), + expected_kind: StoreErrorKind::Busy, + expected_arrival: true, + }, + WorkerDeadlineTestCase { + description: "expired budget sends no request", + budget: Duration::ZERO, + expected_kind: StoreErrorKind::Busy, + expected_arrival: false, + }, + ]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind slow server"); + let url = format!( + "http://{}", + listener.local_addr().expect("slow server address") + ); + let arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let observed = arrived.clone(); + let application = Router::new().route( + "/api/runs/{run_id}/recall", + axum::routing::post(move || { + let arrived = arrived.clone(); + async move { + arrived.store(true, std::sync::atomic::Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(2)).await; + axum::Json(sqlbuild_recall::store::models::InsertOutcome::default()) + } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, application).await }); + let client = connect_worker( + &url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("connect worker to slow server"); + let SessionStore::Hosted(opener) = client.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "deadline".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(1)) + .expect("open HTTP store"); + let started = Instant::now(); + let error = store + .insert(&[], started + test_case.budget) + .expect_err("deadline rejects delivery"); + assert_eq!( + error.kind, test_case.expected_kind, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "{}", + test_case.description + ); + }) + .await + .expect("join slow-server publication"); + assert_eq!( + observed.load(std::sync::atomic::Ordering::SeqCst), + test_case.expected_arrival, + "{}", + test_case.description + ); + task.abort(); + let _ = task.await; + }); + } +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/mod.rs b/crates/sqlbuild-recall/src/records/_helpers/mod.rs index eaf86fcc66..e04c007fa0 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod events; pub(crate) mod ids; +pub(crate) mod record_row; pub(crate) mod rows; pub(crate) mod time; diff --git a/crates/sqlbuild-recall/src/records/_helpers/record_row.rs b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs new file mode 100644 index 0000000000..3a99f89e08 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs @@ -0,0 +1,61 @@ +//! Decode transported record values against the physical schema, then compute their digest. + +use std::collections::BTreeMap; + +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnType, Table}; + +/// Envelope and digest columns belong to the receiver, never the transported values. +pub(crate) fn record_row( + envelope: &RowEnvelope, + table: Table, + values: &BTreeMap, +) -> Result { + let columns = table_spec(table).columns; + let mut decoded = Vec::with_capacity(values.len()); + for (name, value) in values { + let column = columns + .iter() + .find(|column| column.name == name) + .ok_or_else(|| RecordError::new(format!("unknown Recall record column {name}")))?; + let valid = match (value, column.column_type) { + (RecallValue::Null, _) => column.nullable, + (RecallValue::Integer(0 | 1), ColumnType::Boolean) => true, + (RecallValue::Integer(_), ColumnType::Integer) => true, + (RecallValue::Real(value), ColumnType::Real) => value.is_finite(), + (RecallValue::Blob(_), ColumnType::Blob) => true, + (RecallValue::Text(_), ColumnType::Text) => true, + (RecallValue::Text(value), ColumnType::Json) => { + serde_json::from_str::(value).map_err(|error| { + RecordError::new(format!("invalid Recall JSON column {name}: {error}")) + })?; + true + } + (RecallValue::Text(value), ColumnType::Timestamp) => { + chrono::DateTime::parse_from_rfc3339(value).map_err(|error| { + RecordError::new(format!("invalid Recall timestamp column {name}: {error}")) + })?; + true + } + _ => false, + }; + if !valid { + return Err(RecordError::new(format!( + "invalid Recall record column {name}" + ))); + } + decoded.push((column.name, value.clone())); + } + for column in columns.iter().filter(|column| !column.nullable) { + if !values.contains_key(column.name) { + return Err(RecordError::new(format!( + "missing Recall record column {}", + column.name + ))); + } + } + Ok(finish_row(table, envelope, decoded)) +} diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index a7798e3e5c..eaab7e2c60 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -3,7 +3,8 @@ use crate::schema::models::Table; /// One stored value. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] pub enum RecallValue { Null, Integer(i64), @@ -34,6 +35,15 @@ pub struct RecallRow { } impl RecallRow { + /// Validate schema-typed values and rebuild the receiver-owned envelope and content digest. + pub fn from_values( + envelope: &RowEnvelope, + table: Table, + values: &std::collections::BTreeMap, + ) -> Result { + crate::records::_helpers::record_row::record_row(envelope, table, values) + } + /// The value of `column`, if the row has it. #[must_use] pub fn value(&self, column: &str) -> Option<&RecallValue> { diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 5da839a462..e622f273fe 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -91,7 +91,20 @@ pub struct TableSpec { } /// The tables Recall writes. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum Table { Invocations, Output, diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index 61b2f27614..fd53a8b5c7 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -54,7 +54,7 @@ pub enum QuarantineOutcome { } /// A row whose key was already stored with different content; the first row is kept. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct ConflictEvidence { pub table: Table, pub record_key: String, @@ -63,7 +63,9 @@ pub struct ConflictEvidence { } /// What one insert batch did, per table. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] pub struct InsertOutcome { /// Rows written or already present with equal content, by table. pub delivered: Vec<(Table, u64)>, From a61b4e2897d9dc73d484e2c91659a20cc139db49 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:25:37 +0000 Subject: [PATCH 025/138] feat: retain native worker ownership in the project interpreter Verify retained compiler baselines before worker startup and join native heartbeat work on interpreter-side close. Preserve claim identity and report retry keys without treating resource teardown as physical warehouse completion. --- Cargo.lock | 1 + crates/sqlbuild-python/Cargo.toml | 1 + .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 13 ++ .../compiled_project/snapshot_worker.rs | 157 ++++++++++++++++++ fensu.toml | 6 + src/sqlbuild/_native.pyi | 18 ++ .../publication/_test_types.py | 15 ++ .../compiled_project/publication/helpers.py | 85 ++++++++++ .../publication/test_worker.py | 148 +++++++++++++++++ 10 files changed, 445 insertions(+) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py diff --git a/Cargo.lock b/Cargo.lock index c468288a19..ad63bc0ef9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2651,6 +2651,7 @@ dependencies = [ "sqlbuild-state", "stacker", "tokio", + "uuid", ] [[package]] diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 3b066a9b33..270745994d 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -48,6 +48,7 @@ sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true stacker.workspace = true tokio.workspace = true +uuid.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 67cb07e670..7ae7f4d521 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -8,4 +8,5 @@ pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_models; pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_worker; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index cee2c105f2..018f69fb18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; @@ -51,6 +52,17 @@ impl NativeOrchestrationSnapshot { .map_err(|error| PyRuntimeError::new_err(error.to_string())) }) } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } } /// Compile catalog nodes from their owners, never from Python model or graph projections. @@ -139,5 +151,6 @@ pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs new file mode 100644 index 0000000000..966c3709e2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs @@ -0,0 +1,157 @@ +//! A worker interpreter retains the native claim owner; it never opens the control database. + +use std::sync::Mutex; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{PyRef, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; +use uuid::Uuid; + +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; + +struct WorkerOwner { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + session: WorkerSession, +} + +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationWorker { + owner: Mutex>, +} + +impl NativeOrchestrationWorker { + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub(crate) fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) + } +} + +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + fn execution_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + fn report(&self, py: Python<'_>, event_id: &str, change_json: &str) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } +} + +fn unavailable() -> pyo3::PyErr { + PyRuntimeError::new_err("native worker session is closed or unavailable") +} + +impl WorkerOwner { + fn close(self) { + drop(self.session); + drop(self.client); + self.runtime + .shutdown_timeout(std::time::Duration::from_secs(1)); + } +} + +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } +} diff --git a/fensu.toml b/fensu.toml index c06a29ffdd..191fa4912d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1576,6 +1576,12 @@ path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.r symbols = ["NativeOrchestrationSnapshot"] reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only SnapshotInput; binding functions remain checked." +[[targets.rust.rule_exceptions]] +rule = "XSB072" +path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs" +symbols = ["NativeOrchestrationWorker"] +reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only native worker ownership; binding functions remain checked." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6a4f400584..27199c0f53 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1743,6 +1743,24 @@ class NativeOrchestrationSnapshot: def to_json(self) -> str: ... def publish(self, server_url: str, bearer: str) -> str: ... + def start_worker( + self, server_url: str, bearer: str, identity: tuple[str, str] + ) -> NativeOrchestrationWorker: ... + +class NativeOrchestrationWorker: + """A verified claim owner with joined native heartbeat teardown.""" + + def execution_json(self) -> str: ... + def cancel_requested(self) -> bool: ... + def report(self, event_id: str, change_json: str) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, + kind: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: ... def build_orchestration_snapshot( project: CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 8c7301d86d..63a996d791 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -10,9 +10,24 @@ class SnapshotApi(NamedTuple): bearer: str +class ClaimedWorker(NamedTuple): + run_id: str + claim_id: str + bearer: str + + @dataclass(frozen=True) class SnapshotPublicationTestCase: description: str target: str expected_generation: int expected_nodes: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotWorkerTestCase: + description: str + model_file: str + original_sql: str + replacement_sql: str + expected_error: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index cbc2554451..27b44181af 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -20,6 +20,7 @@ from uuid import uuid4 from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, SnapshotApi, ) @@ -93,3 +94,87 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: with urlopen(request, timeout=5) as response: result: dict[str, Any] = json.load(response) return result + + +def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any]) -> ClaimedWorker: + """Provision only the launch boundary; verification and lifecycle reporting use real HTTP.""" + run_id: str = str(uuid4()) + claim_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + selection: list[str] = ["model:orders"] + request: dict[str, Any] = { + "run_id": run_id, + "location_id": api.location_id, + "target": publication["target"], + "snapshot_id": publication["snapshot_id"], + "image_id": None, + "kind": "job", + "origin": "manual", + "job_key": None, + "parent_run_id": None, + "root_run_id": run_id, + "trigger_id": None, + "tick_id": None, + "rerun_of_run_id": None, + "requested_by": "test-admin", + "priority": 0, + "selection": selection, + "write_set": selection, + "settings": {"full_refresh": True}, + "concurrency_groups": [], + "overlap": "allow", + } + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_runs (run_id,location_id,target,snapshot_id,kind,origin,root_run_id," + "requested_by,selection_json,request_json,write_set_digest,status,queued_at," + "node_counts_json,last_event_storage_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + run_id, + api.location_id, + publication["target"], + publication["snapshot_id"], + "job", + "manual", + run_id, + "test-admin", + json.dumps(selection), + json.dumps(request), + hashlib.sha256(json.dumps(selection).encode()).hexdigest(), + "queued", + now.isoformat(), + json.dumps({"pending": 1}), + 0, + ), + ) + _ = connection.execute( + "INSERT INTO sqb_run_nodes (run_id,node_key,attempt,status,writes) " + "VALUES (?,'model:orders',1,'pending',TRUE)", + (run_id,), + ) + _ = connection.execute( + "INSERT INTO sqb_write_claims (target,node_key,run_id,claimed_at) VALUES (?,?,?,?)", + (publication["target"], "model:orders", run_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_run_liveness (run_id,worker_instance,claimed_at,last_heartbeat_at) " + "VALUES (?,?,?,?)", + (run_id, claim_id, now.isoformat(), now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,run_id,claim_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + run_id, + claim_id, + json.dumps({"kind": "run", "run_id": run_id, "claim_id": claim_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + return ClaimedWorker(run_id=run_id, claim_id=claim_id, bearer=bearer) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py new file mode 100644 index 0000000000..b357d3504a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -0,0 +1,148 @@ +"""Retained compiler baselines cross the native worker boundary before lifecycle writes.""" + +import json +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, + SnapshotWorkerTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + claimed_worker, + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotWorkerTestCase( + description="unselected dependency drift", + model_file="models/stg_orders.sql", + original_sql="CAST(5 AS INTEGER)", + replacement_sql="CAST(17 AS INTEGER)", + expected_error="node content differs", + ), + SnapshotWorkerTestCase( + description="selected model drift", + model_file="models/orders.sql", + original_sql="SELECT order_id, amount", + replacement_sql="SELECT order_id, amount * CAST(2 AS BIGINT) AS amount", + expected_error="node content differs", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_claim_lifetime_are_native( + test_case: SnapshotWorkerTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | {"location_id": api.location_id} + baseline: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + publication: dict[str, Any] = json.loads(baseline.publish(api.url, api.bearer)) + claim: ClaimedWorker = claimed_worker(tmp_path / "control", api, publication) + identity: tuple[str, str] = (claim.run_id, claim.claim_id) + changed: dict[str, str] = TYPE_PROOF_FILES | { + test_case.model_file: TYPE_PROOF_FILES[test_case.model_file].replace( + test_case.original_sql, test_case.replacement_sql + ) + } + write_files(project_dir=project_dir, files=changed) + drifted: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + wrong: NativeOrchestrationSnapshot = build_orchestration_snapshot( + drifted, json.dumps(metadata), project_dir + ) + with pytest.raises(RuntimeError, match=test_case.expected_error): + _ = wrong.start_worker(api.url, claim.bearer, identity) + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + location_headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "queued" + with pytest.raises(RuntimeError, match="does not match"): + _ = baseline.start_worker(api.url, claim.bearer, (claim.run_id, str(uuid4()))) + with baseline.start_worker(api.url, claim.bearer, identity) as worker: + execution: dict[str, Any] = json.loads(worker.execution_json()) + assert execution["claim"]["request"]["selection"] == ["model:orders"] + assert execution["claim"]["request"]["settings"] == {"full_refresh": True} + assert frozenset(execution["node_hashes"]) == { + "model:orders", + "model:stg_orders", + } + assert not worker.cancel_requested() + event_id: str = str(uuid4()) + change: str = json.dumps({"kind": "node_started", "node_key": "model:orders"}) + assert worker.report(event_id, change) + assert not worker.report(event_id, change) + assert claim.bearer not in repr(worker) + worker.close() + assert worker.cancel_requested() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.execution_json() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.report(str(uuid4()), change) + with pytest.raises(RuntimeError, match="closed"), worker: + pytest.fail("closed worker cannot be entered") + # Closing an interpreter-side owner is not proof of physical warehouse termination. + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "running" + with baseline.start_worker(api.url, claim.bearer, identity) as resumed: + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "node_finished", + "node_key": "model:orders", + "outcome": "success", + "rows_affected": 7, + "error_code": None, + "skipped_because_of": None, + } + ), + ) + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "finished", + "outcome": "success", + "reason": "worker execution completed", + "exit_code": 0, + } + ), + ) + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + result: dict[str, Any] = json.load(response)[0] + assert result["status"] == "success" + assert result["node_counts_json"] == {"success": 1} From 554051446d3286e1c928bc5c5bc7c4a70f7f6501 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:31:43 +0000 Subject: [PATCH 026/138] refactor: keep scheduler planning entries within their domain Restrict forward and reverse cron expansion and pure tick planning to the scheduler. Preserve scheduling bodies and all existing fault, DST and simulation coverage. --- .../src/scheduler/_helpers/expansion.rs | 4 ++-- .../main/{next_cron_time.rs => _next_cron_time.rs} | 2 +- .../src/scheduler/main/{plan_ticks.rs => _plan_ticks.rs} | 5 ++++- .../main/{previous_cron_time.rs => _previous_cron_time.rs} | 2 +- .../src/scheduler/main/evaluate_trigger.rs | 2 +- crates/sqlbuild-orchestration/src/scheduler/main/mod.rs | 6 +++--- .../src/scheduler/main/next_schedule_time.rs | 2 +- .../sqlbuild-orchestration/src/scheduler/tests/helpers.rs | 2 +- .../src/scheduler/tests/test_backlog.rs | 2 +- .../sqlbuild-orchestration/src/scheduler/tests/test_cron.rs | 4 ++-- .../src/scheduler/tests/test_ticks.rs | 2 +- 11 files changed, 18 insertions(+), 15 deletions(-) rename crates/sqlbuild-orchestration/src/scheduler/main/{next_cron_time.rs => _next_cron_time.rs} (95%) rename crates/sqlbuild-orchestration/src/scheduler/main/{plan_ticks.rs => _plan_ticks.rs} (93%) rename crates/sqlbuild-orchestration/src/scheduler/main/{previous_cron_time.rs => _previous_cron_time.rs} (95%) diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index 8ba7d7e422..75337f66d1 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -6,9 +6,9 @@ use chrono::{DateTime, Utc}; use crate::declarations::models::{Schedule, TriggerDefinition}; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; -use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; use crate::scheduler::main::parse_cron::parse_cron; -use crate::scheduler::main::previous_cron_time::previous_cron_time; use crate::scheduler::models::CatchUp; use crate::scheduler::types::ScheduleExpansion; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs similarity index 95% rename from crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs index 8964653860..90c51039cc 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/next_cron_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs @@ -7,7 +7,7 @@ use crate::scheduler::models::CronSchedule; use crate::store::errors::ControlError; /// Ambiguities select the later fold; missing times move to the end of the actual IANA gap. -pub fn next_cron_time( +pub(in crate::scheduler) fn next_cron_time( schedule: &CronSchedule, after: DateTime, ) -> Result, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs similarity index 93% rename from crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs index 0e8d05b971..842e9308eb 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/plan_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs @@ -10,7 +10,10 @@ use crate::scheduler::models::{TickPlan, TickState}; use crate::store::errors::ControlError; /// Return facts to append and the cursor to commit atomically with them. -pub fn plan_ticks(state: &TickState, now: DateTime) -> Result { +pub(in crate::scheduler) fn plan_ticks( + state: &TickState, + now: DateTime, +) -> Result { if !state.enabled { return Ok(TickPlan { actions: vec![], diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs similarity index 95% rename from crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs index de74a44c70..3297e839ba 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/previous_cron_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs @@ -7,7 +7,7 @@ use crate::scheduler::models::CronSchedule; use crate::store::errors::ControlError; /// Fold and gap mapping is identical to forward expansion, including the later ambiguous instant. -pub fn previous_cron_time( +pub(in crate::scheduler) fn previous_cron_time( schedule: &CronSchedule, at: DateTime, ) -> Result, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs index 0c079450a9..1a2e98d3b7 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs @@ -6,8 +6,8 @@ use uuid::Uuid; use crate::declarations::models::TriggerStatus; use crate::scheduler::_helpers::expansion::expand_schedule; use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::main::check_lease::check_lease; -use crate::scheduler::main::plan_ticks::plan_ticks; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs index 0e3a7afb49..60225c3875 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -1,12 +1,12 @@ +pub(in crate::scheduler) mod _next_cron_time; +pub(in crate::scheduler) mod _plan_ticks; +pub(in crate::scheduler) mod _previous_cron_time; pub mod acquire_lease; pub(crate) mod check_lease; pub mod evaluate_trigger; pub mod list_ticks; -pub mod next_cron_time; pub(crate) mod next_schedule_time; pub mod parse_cron; -pub mod plan_ticks; -pub mod previous_cron_time; pub mod renew_lease; pub mod run_scheduler; pub mod scheduler_cycle; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs index c4675ccbef..880e4ad7c9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs @@ -5,7 +5,7 @@ use std::ops::Bound::{Excluded, Unbounded}; use chrono::{DateTime, Utc}; use crate::declarations::models::Schedule; -use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::_next_cron_time::next_cron_time; use crate::scheduler::main::parse_cron::parse_cron; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs index bfc531a1fc..0484e35cac 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -8,7 +8,7 @@ use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; use proptest::{prop_assert, prop_assert_eq}; use uuid::Uuid; -use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickPlan, TickState}; use crate::scheduler::tests::test_types::TickTestCase; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs index 19798dce2a..ac9985f292 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; use crate::scheduler::_helpers::expansion::expand_schedule; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; -use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap, TickDecision, TickState}; use crate::scheduler::tests::helpers::time; use crate::scheduler::tests::test_types::BacklogTestCase; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs index d05d1e0ae1..fac0537e06 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -1,8 +1,8 @@ use chrono::{DateTime, SecondsFormat}; -use crate::scheduler::main::next_cron_time::next_cron_time; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; use crate::scheduler::main::parse_cron::parse_cron; -use crate::scheduler::main::previous_cron_time::previous_cron_time; use crate::scheduler::tests::test_types::{CronTestCase, PreviousCronTestCase}; #[test] diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs index de1eaeeba9..2aecca4468 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -1,6 +1,6 @@ use std::num::NonZeroUsize; -use crate::scheduler::main::plan_ticks::plan_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap}; use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; use crate::scheduler::tests::test_types::TickTestCase; From 786179f4ee16e66f49ab3b4dc1f8c4275a916753 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:41:55 +0000 Subject: [PATCH 027/138] feat: audit native run cancellation atomically Expose owner-only cancellation with stable retry identity. Commit lifecycle facts, projections and security audit together while retaining active physical claims until termination. --- .../sqlbuild-cli/src/argv/command_table.json | 65 ++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 9 ++- .../src/native_commands/main/_run_admin.rs | 23 ++++++ .../src/native_commands/models.rs | 5 ++ .../tests/native_commands/test_control.rs | 22 ++++++ .../tests/native_commands/worker/helpers.rs | 75 ++++++++++++++----- .../src/runs/_helpers/changes.rs | 6 +- .../src/runs/main/change_run.rs | 26 ++++++- .../sqlbuild-orchestration/src/runs/models.rs | 4 +- .../tests/api/test_contract.rs | 3 +- .../tests/api/test_worker.rs | 8 +- .../tests/runs/test_lifecycle.rs | 74 ++++++++++++++++-- 12 files changed, 290 insertions(+), 30 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index cd0b2193e9..86cf2619e4 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -352,6 +352,66 @@ ] } }, + { + "name": "cancel-run", + "parser": { + "prog": "sqb admin cancel-run", + "description": "Request cancellation through privileged database access; the fact, projection and security audit commit together. Active warehouse claims remain held until execution ends.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "RUN_ID", + "required": true, + "help": "run UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the cancellation fact and security audit", + "group": 1 + } + ] + } + }, { "name": "change-trigger", "parser": { @@ -464,6 +524,11 @@ "metavar": "revoke-token", "help": "revoke a credential by UUID" }, + { + "dest": "cancel-run", + "metavar": "cancel-run", + "help": "request run cancellation with an atomic audit and retry identity" + }, { "dest": "change-trigger", "metavar": "change-trigger", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 0673c8aa18..a04f2333df 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -67,13 +67,20 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { + Some("change-trigger" | "cancel-run") => { let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) .map_err(|_| ControlError::Invalid("--event-id must be a UUID".into()))?; let reason = text(namespace, "admin_reason").unwrap_or_default(); if reason.trim().is_empty() { return Err(ControlError::Invalid("--reason must not be blank".into())); } + if action.as_deref() == Some("cancel-run") { + return Ok(ControlAdmin::CancelRun { + event_id, + run_id: id, + reason, + }); + } let mut change = json!({ "kind": text(namespace, "admin_trigger_change").unwrap_or_default().replace('-', "_"), }); diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 90bc31d618..e51cc02eb3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -8,6 +8,8 @@ use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -42,6 +44,27 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = change_run( + &store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id, + run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason, + }, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "run_id": run_id, "applied": applied})) + } ControlAdmin::ChangeTrigger { event_id, trigger_id, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index b7b6f00b4b..a4847c0813 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -15,6 +15,11 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + CancelRun { + event_id: Uuid, + run_id: Uuid, + reason: String, + }, ChangeTrigger { event_id: Uuid, trigger_id: Uuid, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 74eaf195d3..9c4b553eff 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -35,6 +35,28 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin change-trigger ", expected_stderr: "", }, + CommandTestCase { + description: "native cancellation help", + arguments: &["admin", "cancel-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin cancel-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank cancellation reason rejected before store initialization", + arguments: &[ + "admin", + "cancel-run", + "00000000-0000-0000-0000-000000000001", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, CommandTestCase { description: "blank trigger reason rejected before store initialization", arguments: &[ diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index c951d675f2..dc17641f46 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -23,6 +23,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_orchestration::worker::models::WorkerSession; use uuid::Uuid; +use crate::admin_helpers::administer; use crate::test_types::TestControl; use crate::worker::test_types::WorkerControl; @@ -137,25 +138,63 @@ pub(crate) async fn start_claimed_worker( ) } -pub(crate) fn cancel_worker(worker: WorkerControl, _control: &mut TestControl) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("independent operator runtime"); - runtime - .block_on(change_run( - &worker.store, - RunAuthority::Operator("test-admin"), - &RunChangeRequest { - event_id: Uuid::now_v7(), - run_id: worker.identity.0, - claim_id: Some(worker.identity.1), - change: RunChange::CancelRequested { - actor: "test-admin".into(), - }, +pub(crate) fn cancel_worker(worker: WorkerControl, control: &mut TestControl) { + let event_id = Uuid::now_v7().to_string(); + let run_id = worker.identity.0.to_string(); + let arguments = [ + "cancel-run", + &run_id, + "--event-id", + &event_id, + "--reason", + "Stop the orders build", + ]; + let applied = administer(control, &arguments); + assert_eq!(applied["run_id"], run_id); + assert_eq!(applied["event_id"], event_id); + assert_eq!(applied["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable cancellation audit"); + connection + .query_row( + "SELECT actor, granting_role, scope_json, before_json, after_json FROM sqb_security_audit WHERE event_id=?1 AND action='run.cancel_requested'", + [&event_id], + |row| { + let actor: String = row.get(0)?; + let role: String = row.get(1)?; + let scope: String = row.get(2)?; + let before: String = row.get(3)?; + let after: String = row.get(4)?; + let scope: serde_json::Value = serde_json::from_str(&scope).expect("audit scope"); + let before: serde_json::Value = serde_json::from_str(&before).expect("before state"); + let after: serde_json::Value = serde_json::from_str(&after).expect("after state"); + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert_eq!(scope["run_id"], run_id); + assert_eq!(scope["target"], "development"); + assert!(scope["location_id"].is_string()); + assert_eq!(before["status"], "running"); + assert!(before["cancel_requested_at"].is_null()); + assert_eq!(after["status"], "running"); + assert!(after["cancel_requested_at"].is_string()); + assert_eq!(after["reason"], "Stop the orders build"); + Ok(()) }, - )) - .expect("request cancellation while worker runtime is blocked"); + ) + .expect("same-transaction operator history"); + let claims: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqb_write_claims WHERE run_id=?1", + [&run_id], + |row| row.get(0), + ) + .expect("physical claims after cancellation request"); + assert_eq!( + claims, 1, + "cancellation is not proof of physical termination" + ); } pub(crate) fn revoke_worker(worker: WorkerControl, _control: &mut TestControl) { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs index e9480d69da..6c9fcee455 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs @@ -55,8 +55,10 @@ pub(crate) fn validate_authority( } ) } - RunAuthority::Operator(actor) => { - matches!(&request.change, RunChange::CancelRequested { actor: recorded } if !actor.trim().is_empty() && actor == recorded) + RunAuthority::Operator(context) => { + matches!(&request.change, RunChange::CancelRequested { actor, reason } + if !context.actor.trim().is_empty() && context.actor == *actor + && !context.granting_role.trim().is_empty() && !reason.trim().is_empty()) } }; if !allowed { diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 02fe0308c1..012ff4dbe2 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -2,9 +2,10 @@ use serde_json::json; +use crate::credentials::main::append_audit::append_audit; use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; use crate::runs::_helpers::publish_change::publish_change; -use crate::runs::models::{ChangingRun, RunAuthority, RunChangeRequest}; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -50,6 +51,29 @@ pub async fn change_run( )?; validate_change(&state, request)?; let mut transaction = publish_change(transaction, request, &state).await?; + if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = + (authority, &request.change) + { + let mut rows; + (transaction, rows) = query( + transaction, + "SELECT location_id, target, status, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let after = rows.remove(0); + transaction = append_audit( + transaction, + context, + (request.event_id, "run.cancel_requested"), + ( + json!({"location_id": after["location_id"], "target": after["target"], "run_id": request.run_id}), + json!({"status": state.status, "cancel_requested_at": state.cancel_requested_at}), + json!({"status": after["status"], "cancel_requested_at": after["cancel_requested_at"], "reason": reason}), + ), + ) + .await?; + } if let RunAuthority::Scheduler(lease) = authority { transaction = check_lease(transaction, lease).await?; } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index c0496bd605..643c126459 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -211,6 +211,8 @@ pub enum RunChange { Recovered, CancelRequested { actor: String, + #[serde(default)] + reason: String, }, Finished { outcome: RunOutcome, @@ -248,7 +250,7 @@ pub struct RunChangeRequest { pub enum RunAuthority<'a> { Scheduler(&'a crate::scheduler::models::SchedulerLease), Worker { run_id: Uuid, claim_id: Uuid }, - Operator(&'a str), + Operator(&'a crate::credentials::models::AuditContext), } #[derive(Clone, Debug, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index df890b2024..65ad349b97 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -419,6 +419,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie event_id: Uuid::now_v7(), change: RunChange::CancelRequested { actor: "test-admin".into(), + reason: "Stop the orders build".into(), }, ..started.clone() }; @@ -447,7 +448,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .expect("heartbeat response"), json!({"cancel_requested": false}) ); - change_run(store, RunAuthority::Operator("test-admin"), &cancellation) + change_run(store, RunAuthority::Operator(&audit), &cancellation) .await .expect("operator cancellation"); let response: serde_json::Value = client diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 07a1beb282..8f4bf9e993 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -282,13 +282,19 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a .expect("another launched process"); change_run( store, - RunAuthority::Operator("test-operator"), + RunAuthority::Operator(&AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }), &RunChangeRequest { event_id: Uuid::now_v7(), run_id: cancelled_identity.0, claim_id: Some(cancelled_identity.1), change: RunChange::CancelRequested { actor: "test-operator".into(), + reason: "Stop before execution".into(), }, }, ) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index d06a397736..8dca64c004 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; @@ -75,9 +76,15 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ run_id: Uuid::from_u128(2), claim_id: Uuid::now_v7(), }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; assert!(change_run(store, other_worker, &started).await.is_err()); assert!( - change_run(store, RunAuthority::Operator("test-operator"), &started) + change_run(store, RunAuthority::Operator(&operator), &started) .await .is_err() ); @@ -268,6 +275,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ event_id: Uuid::now_v7(), change: RunChange::CancelRequested { actor: "test-operator".into(), + reason: "Stop the orders build".into(), }, ..started.clone() }; @@ -275,7 +283,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ change_run(store, worker, &cancel).await.is_err(), "workers cannot impersonate operators" ); - change_run(store, RunAuthority::Operator("test-operator"), &cancel) + change_run(store, RunAuthority::Operator(&operator), &cancel) .await .expect("record cancellation"); assert!( @@ -386,23 +394,79 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l claim_id: None, change: RunChange::CancelRequested { actor: "test-operator".into(), + reason: "Cancel queued orders".into(), }, }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let other = AuditContext { + actor: "another-operator".into(), + ..operator.clone() + }; assert!( - change_run(store, RunAuthority::Operator("another-operator"), &request) + change_run(store, RunAuthority::Operator(&other), &request) .await .is_err() ); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("facts before audit failure"); + let point = TransactionFault { + description: "cancellation audit failure rolls back terminal facts and projections", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cancellation audit failure')", + postgres_body: "RAISE EXCEPTION 'injected cancellation audit failure'", + }; + install_fault(store, &point).await; assert!( - change_run(store, RunAuthority::Operator("test-operator"), &request) + change_run(store, RunAuthority::Operator(&operator), &request) + .await + .is_err(), + "{}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("cancel rollback") + .remove(0) + .status, + RunStatus::Queued + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before_events + ); + assert!( + change_run(store, RunAuthority::Operator(&operator), &request) .await .expect("cancel before launch") ); assert!( - !change_run(store, RunAuthority::Operator("test-operator"), &request) + !change_run(store, RunAuthority::Operator(&operator), &request) .await .expect("retry cancellation") ); + let mut changed = request.clone(); + changed.change = RunChange::CancelRequested { + actor: operator.actor.clone(), + reason: "Different cancellation reason".into(), + }; + assert!( + change_run(store, RunAuthority::Operator(&operator), &changed) + .await + .is_err(), + "retry identity includes the cancellation reason" + ); let lease = acquire_lease(store, Uuid::now_v7()) .await .expect("elect") From e2e767c5044d1645e7d8d705a8d9c624f368ddc9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:58:25 +0000 Subject: [PATCH 028/138] feat: queue pinned published jobs with atomic audits Resolve manual requests against the current published job inside one write transaction. Preserve the accepted snapshot and execution inputs across retries after redeployment, and commit queue publication with the trusted owner audit. --- .../sqlbuild-cli/src/argv/command_table.json | 94 ++++++++ .../src/native_commands/_helpers/decision.rs | 33 ++- .../src/native_commands/main/_run_admin.rs | 5 + .../src/native_commands/models.rs | 2 + .../tests/native_commands/test_control.rs | 45 ++++ .../tests/native_commands/worker/helpers.rs | 68 +++--- .../src/runs/_helpers/job_request.rs | 63 +++++ .../src/runs/_helpers/mod.rs | 2 + .../src/runs/_helpers/request.rs | 66 ++++++ .../src/runs/main/job_write_set.rs | 30 +++ .../src/runs/main/mod.rs | 2 + .../src/runs/main/request_job_run.rs | 68 ++++++ .../src/runs/main/request_run.rs | 63 +---- .../sqlbuild-orchestration/src/runs/models.rs | 12 + .../src/scheduler/_helpers/persistence.rs | 20 +- crates/sqlbuild-orchestration/tests/runs.rs | 2 + .../tests/runs/test_requests.rs | 219 ++++++++++++++++++ 17 files changed, 691 insertions(+), 103 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/request.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_requests.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 86cf2619e4..7d04786726 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -352,6 +352,95 @@ ] } }, + { + "name": "request-run", + "parser": { + "prog": "sqb admin request-run", + "description": "Queue a published job using privileged database access. Selection, image, settings and write claims come from the snapshot; the request and audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_job", + "metavar": "JOB", + "required": true, + "help": "published job name", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--run-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--priority" + ], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, { "name": "cancel-run", "parser": { @@ -524,6 +613,11 @@ "metavar": "revoke-token", "help": "revoke a credential by UUID" }, + { + "dest": "request-run", + "metavar": "request-run", + "help": "queue a pinned published job with an atomic security audit" + }, { "dest": "cancel-run", "metavar": "cancel-run", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index a04f2333df..9e79832001 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, Utc}; use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::runs::models::JobRunRequest; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -67,13 +68,41 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { + Some("change-trigger" | "cancel-run" | "request-run") => { + let identity_option = if action.as_deref() == Some("request-run") { + "--run-id" + } else { + "--event-id" + }; let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) - .map_err(|_| ControlError::Invalid("--event-id must be a UUID".into()))?; + .map_err(|_| ControlError::Invalid(format!("{identity_option} must be a UUID")))?; let reason = text(namespace, "admin_reason").unwrap_or_default(); if reason.trim().is_empty() { return Err(ControlError::Invalid("--reason must not be blank".into())); } + if action.as_deref() == Some("request-run") { + let target = text(namespace, "admin_target").unwrap_or_default(); + let job_key = text(namespace, "admin_job").unwrap_or_default(); + if target.trim().is_empty() || job_key.trim().is_empty() { + return Err(ControlError::Invalid( + "target and job must not be blank".into(), + )); + } + let priority = match namespace.get("admin_priority") { + Some(Value::Int(value)) => value.parse::().map_err(|_| { + ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) + })?, + _ => 0, + }; + return Ok(ControlAdmin::RequestJobRun(JobRunRequest { + run_id: event_id, + location_id: id, + target, + job_key, + reason, + priority, + })); + } if action.as_deref() == Some("cancel-run") { return Ok(ControlAdmin::CancelRun { event_id, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index e51cc02eb3..9c2bbde856 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -9,6 +9,7 @@ use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -44,6 +45,10 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = request_job_run(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } ControlAdmin::CancelRun { event_id, run_id, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index a4847c0813..7d7cbc3df5 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::runs::models::JobRunRequest; use uuid::Uuid; use crate::argv::models::Namespace; @@ -15,6 +16,7 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + RequestJobRun(JobRunRequest), CancelRun { event_id: Uuid, run_id: Uuid, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 9c4b553eff..2347a94d9a 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -42,6 +42,51 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin cancel-run ", expected_stderr: "", }, + CommandTestCase { + description: "native manual job request help", + arguments: &["admin", "request-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin request-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank job rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + " ", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and job must not be blank\n", + }, + CommandTestCase { + description: "oversized manual queue priority rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + "daily_orders", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--priority", + "9223372036854775808", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --priority must fit a signed 64-bit integer\n", + }, CommandTestCase { description: "blank cancellation reason rejected before store initialization", arguments: &[ diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index dc17641f46..2948290095 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -9,12 +9,8 @@ use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunKind, RunOrigin, RunRequest, -}; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use sqlbuild_orchestration::store::main::open_store::open_store; @@ -64,33 +60,40 @@ pub(crate) async fn start_claimed_worker( python_source: None, }, )]), - documents: BTreeMap::new(), + documents: BTreeMap::from([( + "jobs".into(), + serde_json::json!({"daily_orders": { + "description": "Build orders", "selection": ["orders"], "overlap": "allow", "settings": {"full_refresh": true} + }}), + )]), }; - let publication = publish_snapshot(&store, &baseline).await.expect("snapshot"); + publish_snapshot(&store, &baseline).await.expect("snapshot"); let run_id = Uuid::now_v7(); - let request = RunRequest { - run_id, - location_id, - target: "development".into(), - snapshot_id: publication.snapshot_id, - image_id: None, - kind: RunKind::Adhoc, - origin: RunOrigin::Manual, - job_key: None, - parent_run_id: None, - root_run_id: run_id, - trigger_id: None, - tick_id: None, - rerun_of_run_id: None, - requested_by: "test-admin".into(), - priority: 0, - selection: ["orders".into()].into(), - write_set: ["orders".into()].into(), - settings: BTreeMap::new(), - concurrency_groups: Default::default(), - overlap: Overlap::Allow, - }; - request_run(&store, &request).await.expect("queue run"); + let location = location_id.to_string(); + let run = run_id.to_string(); + let arguments = [ + "request-run", + &location, + "daily_orders", + "--target", + "development", + "--run-id", + &run, + "--priority", + "-3", + "--reason", + "Refresh orders", + ]; + let requested = administer(control, &arguments); + assert_eq!(requested["run_id"], run); + assert_eq!(requested["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = rusqlite::Connection::open(&path).expect("audit connection"); + let count: i64 = connection.query_row( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='job.run_requested' AND actor='local-control-store-owner' AND granting_role='control-store-owner' AND json_extract(scope_json,'$.run_id')=?1", + [&run], |row| row.get(0)).expect("trusted manual request audit"); + assert_eq!(count, 1); + drop(connection); let lease = acquire_lease(&store, Uuid::now_v7()) .await .expect("acquire lease") @@ -99,6 +102,11 @@ pub(crate) async fn start_claimed_worker( .await .expect("claim") .expect("queued work"); + assert_eq!(claim.request.priority, -3); + assert_eq!( + claim.request.settings.get("full_refresh"), + Some(&serde_json::json!(true)) + ); let identity = (run_id, claim.claim_id); let token = issue_token( &store, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs new file mode 100644 index 0000000000..691436c7c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs @@ -0,0 +1,63 @@ +//! Freeze execution inputs from one published job under the caller's write transaction. + +use crate::credentials::models::AuditContext; +use crate::declarations::models::JobDefinition; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_job_request( + transaction: ControlTransaction, + context: &AuditContext, + request: &JobRunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (transaction, jobs) = query( + transaction, + "SELECT j.snapshot_id, j.definition_json, s.image_id FROM sqb_jobs j \ + JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3 AND j.kind='job'", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], + ).await?; + let row = jobs.first().ok_or_else(|| { + ControlError::Invalid("published job not found in this location and target".into()) + })?; + let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let image_id = serde_json::from_value(row["image_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = job_write_set( + transaction, + snapshot_id, + (request.location_id, &request.target, &request.job_key), + ) + .await?; + Ok(( + transaction, + RunRequest { + run_id: request.run_id, + location_id: request.location_id, + target: request.target.clone(), + snapshot_id, + image_id, + kind: RunKind::Job, + origin: RunOrigin::Manual, + job_key: Some(request.job_key.clone()), + parent_run_id: None, + root_run_id: request.run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: context.actor.clone(), + priority: request.priority, + selection: definition.selection, + write_set, + settings: definition.settings, + concurrency_groups: definition.concurrency_groups, + overlap: definition.overlap, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index a3adbd20a7..a8a8e0a6f5 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -1,8 +1,10 @@ pub(crate) mod changes; pub(crate) mod claims; pub(crate) mod events; +pub(crate) mod job_request; pub(crate) mod nodes; pub(crate) mod output; pub(crate) mod projection; pub(crate) mod publish_change; pub(crate) mod recall; +pub(crate) mod request; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs new file mode 100644 index 0000000000..b4b727be05 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs @@ -0,0 +1,66 @@ +//! Validate and publish a frozen request inside the caller's audit transaction. + +use std::collections::BTreeSet; + +use serde_json::json; + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_request( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.requested_by.trim().is_empty() + || request.selection.is_empty() + || !request.write_set.is_subset(&request.selection) + { + return Err(ControlError::Invalid( + "run needs an actor, selection and a contained write set".into(), + )); + } + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&json!(request)) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + return Ok((transaction, false)); + } + let (transaction, nodes) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ + (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3 \ + AND image_id IS NOT DISTINCT FROM $4)", + vec![ + Parameter::Id(request.snapshot_id), + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::OptionalText(request.image_id.clone()), + ], + ) + .await?; + let keys: BTreeSet<&str> = nodes + .iter() + .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) + .collect(); + if request + .selection + .iter() + .any(|key| !keys.contains(key.as_str())) + { + return Err(ControlError::Invalid( + "run selection or image does not match the pinned location and target snapshot".into(), + )); + } + Ok((queue_run(transaction, request).await?, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs new file mode 100644 index 0000000000..1354b035fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -0,0 +1,30 @@ +//! Resolve physical writers from the pinned snapshot and its published job membership. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn job_write_set( + transaction: ControlTransaction, + snapshot_id: Uuid, + (location_id, target, job_key): (Uuid, &str, &str), +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ + WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", + vec![Parameter::Id(snapshot_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(job_key.into())]).await?; + Ok(( + transaction, + rows.into_iter() + .filter_map(|row| { + row.get("node_key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }) + .collect(), + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index da3eca538f..97d86e1251 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,11 +1,13 @@ pub mod change_run; pub mod claim_next_run; pub mod heartbeat_run; +pub(crate) mod job_write_set; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; +pub mod request_job_run; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs new file mode 100644 index 0000000000..e768a67f4d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -0,0 +1,68 @@ +//! Direct-store owners queue published jobs without supplying or widening their write scope. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::job_request::frozen_job_request; +use crate::runs::_helpers::request::publish_request; +use crate::runs::models::JobRunRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The context is a trusted control-store owner, not a role supplied by an HTTP caller. +pub async fn request_job_run( + store: &ControlStore, + context: &AuditContext, + request: &JobRunRequest, +) -> Result { + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.target.trim().is_empty() + || request.job_key.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "job run needs an actor, granting role, target, job and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", + vec![Parameter::Id(event_id), Parameter::Id(request.run_id)], + ).await?; + if let Some(previous) = previous.first() { + if previous.get("actor") != Some(&json!(context.actor)) + || previous + .get("after_json") + .and_then(|after| after.get("request")) + != Some(&json!(request)) + { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, frozen) = frozen_job_request(transaction, context, request).await?; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + context, + (event_id, "job.run_requested"), + ( + json!({"location_id": request.location_id, "target": request.target, "job_key": request.job_key, "run_id": request.run_id}), + serde_json::Value::Null, + json!({"request": request, "frozen": frozen}), + ), + ).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs index 82616a6ee2..71665e84da 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -1,69 +1,16 @@ //! Queue a frozen, already authorized request without recreating work on transport retries. -use std::collections::BTreeSet; - -use serde_json::json; - -use crate::runs::main::queue_run::queue_run; +use crate::runs::_helpers::request::publish_request; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// The API must compute and authorize this selection; callers cannot supply their own write scope. pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result { - if request.requested_by.trim().is_empty() - || request.selection.is_empty() - || !request.write_set.is_subset(&request.selection) - { - return Err(ControlError::Invalid( - "run needs an actor, selection and a contained write set".into(), - )); - } let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = query( - transaction, - "SELECT request_json FROM sqb_runs WHERE run_id=$1", - vec![Parameter::Id(request.run_id)], - ) - .await?; - if let Some(previous) = previous.first() { - if previous.get("request_json") != Some(&json!(request)) { - return Err(ControlError::Invalid( - "run ID already has different request content".into(), - )); - } - commit(transaction).await?; - return Ok(false); - } - let (transaction, nodes) = query( - transaction, - "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ - (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3 \ - AND image_id IS NOT DISTINCT FROM $4)", - vec![ - Parameter::Id(request.snapshot_id), - Parameter::Id(request.location_id), - Parameter::Text(request.target.clone()), - Parameter::OptionalText(request.image_id.clone()), - ], - ) - .await?; - let keys: BTreeSet<&str> = nodes - .iter() - .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) - .collect(); - if request - .selection - .iter() - .any(|key| !keys.contains(key.as_str())) - { - return Err(ControlError::Invalid( - "run selection or image does not match the pinned location and target snapshot".into(), - )); - } - commit(queue_run(transaction, request).await?).await?; - Ok(true) + let (transaction, applied) = publish_request(transaction, request).await?; + commit(transaction).await?; + Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 643c126459..70a560ab44 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -88,6 +88,18 @@ pub struct RunRequest { pub overlap: crate::scheduler::models::Overlap, } +/// A trusted operator names a published job; execution inputs are never supplied by the caller. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobRunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub job_key: String, + pub reason: String, + pub priority: i64, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index e7c466d77b..e84c4f73b9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -1,8 +1,8 @@ //! Deterministic tick insertion gates all run facts; retries cannot queue a second run. use serde_json::json; -use std::collections::BTreeSet; +use crate::runs::main::job_write_set::job_write_set; use crate::runs::main::queue_run::queue_run; use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; @@ -16,18 +16,12 @@ pub(crate) async fn persist_ticks( plan: &TickPlan, ) -> Result { let now = transaction.now; - let (mut transaction, rows) = query(transaction, - "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ - WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", - vec![Parameter::Id(trigger.snapshot_id), Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; - let write_set: BTreeSet = rows - .into_iter() - .filter_map(|row| { - row.get("node_key") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) - }) - .collect(); + let (mut transaction, write_set) = job_write_set( + transaction, + trigger.snapshot_id, + (trigger.location_id, &trigger.target, &trigger.job_key), + ) + .await?; for action in &plan.actions { let (outcome, run, covered) = match action.decision { TickDecision::Requested { run_id } => ("requested", Some(run_id), None), diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index 2ab841868c..b2b4e61852 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -28,5 +28,7 @@ mod test_faults; mod test_lifecycle; #[path = "runs/test_monitor.rs"] mod test_monitor; +#[path = "runs/test_requests.rs"] +mod test_requests; #[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs new file mode 100644 index 0000000000..73424f6c21 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -0,0 +1,219 @@ +//! Published-job requests freeze execution inputs and audit atomically on both databases. + +use std::collections::BTreeMap; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RunStatus}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::location_helpers::create_location; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; + +#[test] +fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_and_frozen_execution_commit_once() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite queues and audits the pinned job exactly once", + backend: Backend::Sqlite, + expected_status: RunStatus::Queued, + }, + RunLifecycleTestCase { + description: "PostgreSQL queues and audits the pinned job exactly once", + backend: Backend::Postgres, + expected_status: RunStatus::Queued, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location_id); + let mut source = input.nodes["orders"].clone(); + source.kind = "source".into(); + input.nodes.insert("warehouse_orders".into(), source); + input.documents.insert("jobs".into(), json!({"daily_orders": { + "description": "Build orders from the warehouse source", "selection": ["orders", "warehouse_orders"], + "settings": {"full_refresh": true}, "concurrency_groups": ["orders-load"], "overlap": "queue" + }})); + let published = publish_snapshot(store, &input).await.expect("publish job"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh orders".into(), + priority: 7, + }; + for invalid in [ + JobRunRequest { + target: "production".into(), + ..request.clone() + }, + JobRunRequest { + location_id: Uuid::now_v7(), + ..request.clone() + }, + JobRunRequest { + job_key: "unknown_job".into(), + ..request.clone() + }, + JobRunRequest { + reason: " ".into(), + ..request.clone() + }, + ] { + assert!(request_job_run(store, &context, &invalid).await.is_err()); + } + let fault = TransactionFault { + description: "manual run audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_job_run(store, &context, &request).await.is_err()); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location_id, "development"), 10) + .await + .expect("rolled back projection") + .is_empty() + ); + assert!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back facts") + .is_empty() + ); + let (first, retry) = tokio::join!( + request_job_run(store, &context, &request), + request_job_run(store, &context, &request) + ); + assert_ne!(first.expect("first request"), retry.expect("concurrent retry")); + let facts = read_recall_events(store, 0, 100) + .await + .expect("request facts"); + assert_eq!(facts.len(), 2); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["settings"] = + json!({"full_refresh": false}); + publish_snapshot(store, &input).await.expect("redeploy"); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry keeps original snapshot") + ); + let mut changed = request.clone(); + changed.reason = "Different request".into(); + assert!(request_job_run(store, &context, &changed).await.is_err()); + let other_actor = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!( + request_job_run(store, &other_actor, &request) + .await + .is_err() + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no duplicate facts"), + facts + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "orders-load".into(), + std::num::NonZeroU32::new(1).expect("positive group limit"), + )]); + let claim = claim_next_run(store, &lease, &limits) + .await + .expect("claim") + .expect("queued request"); + assert_eq!(claim.request.run_id, request.run_id); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!( + claim.request.image_id.as_deref(), + Some("sha256:synthetic-orders-image") + ); + assert_eq!( + claim.request.selection, + ["orders".into(), "warehouse_orders".into()].into() + ); + assert_eq!(claim.request.write_set, ["orders".into()].into()); + assert_eq!( + claim.request.settings, + BTreeMap::from([("full_refresh".into(), json!(true))]) + ); + assert_eq!( + claim.request.concurrency_groups, + ["orders-load".into()].into() + ); + assert_eq!(claim.request.priority, 7); + assert_eq!(claim.request.requested_by, "test-operator"); + let fresh = JobRunRequest { + run_id: Uuid::now_v7(), + ..request.clone() + }; + assert!( + request_job_run(store, &context, &fresh) + .await + .expect("new request uses redeploy") + ); + let runs = list_runs(store, (location_id, "development"), 10) + .await + .expect("queued jobs"); + assert_eq!(runs.len(), 2); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .selection_json, + ["inventory".into()].into() + ); + close_test_store(fixture).await; + }); + } +} From 6d4ac364a982376754bc250d34353568e9b80e95 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:18:39 +0000 Subject: [PATCH 029/138] feat: publish compiled locations through scoped HTTP Compile in the project interpreter and retain native snapshots through publication. Keep JSON results separate from progress and distinguish uncertain acknowledgements from failed result delivery. --- .../sqlbuild-cli/src/argv/command_table.json | 88 ++++++++ crates/sqlbuild-cli/src/dispatch/constants.rs | 18 ++ .../src/dispatch/main/command_dispatch.rs | 7 +- .../src/command_connection/constants.rs | 3 +- .../sqlbuild-core/src/progress/constants.rs | 1 + .../src/progress/main/machine_output_mode.rs | 3 +- .../tests/runs/test_requests.rs | 5 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 + .../_helpers/compiled_project/registration.rs | 1 + .../_helpers/compiled_project/snapshot.rs | 33 ++- .../_helpers/location_command/command.rs | 199 ++++++++++++++++++ .../bindings/_helpers/location_command/mod.rs | 1 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/_helpers/native_host/host.rs | 8 + .../bindings/_helpers/records/constants.rs | 2 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 12 ++ .../compiled_project/publication/helpers.py | 8 + .../publication/test_publication.py | 154 ++++++++++++++ 19 files changed, 533 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 7d04786726..cd3d7e216a 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,89 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "location", + "parser": { + "prog": "sqb location", + "description": "Compile and publish a project from its own Python environment. Uses SQLBUILD_LOCATION_TOKEN for scoped HTTP access, never the control database. Results are JSON; progress goes to stderr.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "location_command", + "required": true, + "choices": ["publish"], + "help": "compile the project and publish one immutable snapshot", + "group": 0 + }, + { + "kind": "store", + "options": ["--server"], + "dest": "location_server_url", + "required": true, + "help": "control API HTTP or HTTPS origin, without credentials", + "group": 1 + }, + { + "kind": "store", + "options": ["--location-id"], + "dest": "location_id", + "required": true, + "help": "registered location UUID", + "group": 1 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "target", + "required": true, + "help": "configured project target to compile", + "group": 1 + }, + { + "kind": "store", + "options": ["--generation"], + "dest": "location_generation", + "type": "int", + "required": true, + "help": "positive generation newer than the last successful publication", + "group": 1 + }, + { + "kind": "store", + "options": ["--image-id"], + "dest": "location_image_id", + "help": "immutable deployment image identity to pin for workers", + "group": 1 + }, + { + "kind": "store", + "options": ["--project-dir", "--sqb-project-dir"], + "dest": "project_dir", + "default": {"suppress": true}, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + } + ] + } + }, { "name": "admin", "parser": { @@ -8274,6 +8357,11 @@ } ], "choices_help": [ + { + "dest": "location", + "metavar": "location", + "help": "compile and publish code-location snapshots over HTTP" + }, { "dest": "admin", "metavar": "admin", diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 550b31ebe3..e224c4f9f4 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,6 +515,24 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); +const LOCATION_ARGUMENTS: [Field; 6] = [ + field("project_dir", Source::EffectiveProjectDir), + SELECTED_TARGET, + same("location_id"), + attr("server_url", "location_server_url"), + attr("generation", "location_generation"), + attr("image_id", "location_image_id"), +]; +pub(crate) const LOCATION_ROUTE: Route = routed( + "run_location", + "run_location_command", + &LOCATION_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const JANITOR_FIELDS: [Field; 11] = [ PROJECT_DIR, NO_COLOR, diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 29de9f0374..bb91c6290a 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,9 +6,9 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, RECALL_COMMAND_NAME, - RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, - TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, + RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, + SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -55,6 +55,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { RECALL_COMMAND_NAME => run(&RECALL_ROUTE), "cost" => run(&COST_ROUTE), "debug" => run(&DEBUG_ROUTE), + "location" => run(&LOCATION_ROUTE), "janitor" => run(&JANITOR_ROUTE), "init" => run(&INIT_ROUTE), "playground" => run(&PLAYGROUND_ROUTE), diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index f5c5246f56..7a68e32d8a 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -24,7 +24,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -50,6 +50,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ ("format", None), ("init", None), ("lineage", None), + ("location", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 36ee9d8e7e..bcdf9d0799 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -110,6 +110,7 @@ pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; pub(crate) const EXECUTION_TITLE: &str = "Execution"; pub(crate) const LINEAGE_COMMAND: &str = "lineage"; pub(crate) const QUERY_COMMAND: &str = "query"; +pub(crate) const LOCATION_COMMAND: &str = "location"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs index 7a6c3bc531..1337d46807 100644 --- a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -1,7 +1,7 @@ //! Whether a command's stdout carries machine output, so human progress must use stderr. use crate::progress::constants::{ - DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, + DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, LOCATION_COMMAND, QUERY_COMMAND, }; use crate::progress::models::OutputModeFacts; @@ -11,6 +11,7 @@ pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); match facts.command.as_deref() { _ if facts.json => true, + Some(LOCATION_COMMAND) => true, Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, Some(QUERY_COMMAND) => format(&facts.query_format) == JSON_FORMAT, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 73424f6c21..0cf4c4b234 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -116,7 +116,10 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an request_job_run(store, &context, &request), request_job_run(store, &context, &request) ); - assert_ne!(first.expect("first request"), retry.expect("concurrent retry")); + assert_ne!( + first.expect("first request"), + retry.expect("concurrent retry") + ); let facts = read_recall_events(store, 0, 100) .await .expect("request facts"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f92b9941c0..1f05467222 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -23,6 +23,7 @@ use crate::bindings::_helpers::init_command::command::run_init_command; use crate::bindings::_helpers::janitor::janitor_command::run_janitor_command; use crate::bindings::_helpers::lineage::lineage_command::run_lineage_command; use crate::bindings::_helpers::load::load_command::run_load_command; +use crate::bindings::_helpers::location_command::command::run_location_command; use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; @@ -387,6 +388,7 @@ fn run_implementation<'py>( "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, "run_load_command" => run_load_command(py, &request()?)?, + "run_location_command" => run_location_command(py, options())?, "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index c342479217..2c8daa3464 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -7,5 +7,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; input_loops::registration::register(module)?; compiled_views::registration::register(module)?; + crate::bindings::_helpers::location_command::command::register(module)?; crate::bindings::_helpers::compiled_project::snapshot::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 018f69fb18..a2f44bf10a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -10,8 +10,9 @@ use pyo3::prelude::{ use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use serde_json::Value; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ AuthoredJob, compile_declarations, discover_declarations, @@ -30,6 +31,26 @@ pub(crate) struct NativeOrchestrationSnapshot { pub(crate) input: SnapshotInput, } +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub(crate) fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} + #[pymethods] impl NativeOrchestrationSnapshot { /// Serialize only at the public boundary; native transports can use the retained input. @@ -40,14 +61,10 @@ impl NativeOrchestrationSnapshot { /// Publish retained data without projecting it through Python, releasing the GIL during IO. fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - let record = result.map_err(|error| PyRuntimeError::new_err(error.to_string()))?; serde_json::to_string(&record) .map_err(|error| PyRuntimeError::new_err(error.to_string())) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs new file mode 100644 index 0000000000..001f7211e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -0,0 +1,199 @@ +//! Compile in the location interpreter and publish data through its scoped HTTP credential. + +use std::path::Path; +use std::process::{Command, Stdio}; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; +use crate::bindings::_helpers::compiled_project::snapshot::{ + NativeOrchestrationSnapshot, build_orchestration_snapshot, +}; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::host::write_console; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +fn status(py: Python<'_>, message: &str) -> PyResult<()> { + write_console(py, "stderr", &format!("{message}\n")) +} + +/// Git is optional in deployed images; absent metadata is not fabricated from the last publication. +fn metadata( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + directory: &Path, +) -> PyResult { + let location: String = arguments.value("location_id")?.extract()?; + let location_id = location + .parse() + .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; + let target: String = arguments.value("selected_target")?.extract()?; + let generation: i64 = arguments.value("generation")?.extract()?; + if target.trim().is_empty() || generation < 1 { + return Err(PyValueError::new_err( + "target must not be blank and generation must be positive", + )); + } + let (git_revision, git_dirty, git_author) = py.detach(|| -> PyResult<_> { + let revision = head_commit(directory); + let mut dirty = false; + let mut author = None; + if revision.is_some() { + for (is_dirty, options) in [ + (false, vec!["log", "-1", "--format=%an", "HEAD"]), + ( + true, + vec!["status", "--porcelain", "--untracked-files=normal"], + ), + ] { + let output = Command::new("git") + .current_dir(directory) + .args(options) + .stdin(Stdio::null()) + .output()?; + if !output.status.success() { + return Err(PyRuntimeError::new_err( + "could not read location Git metadata", + )); + } + let text = String::from_utf8_lossy(&output.stdout); + if is_dirty { + dirty = !text.trim().is_empty(); + } else { + author = Some(text.trim().to_owned()); + } + } + } + Ok((revision, dirty, author)) + })?; + Ok(SnapshotInput { + location_id, + target, + generation, + git_revision, + git_dirty, + git_author, + image_id: arguments.value("image_id")?.extract()?, + environment_hash: String::new(), + sqb_version: String::new(), + nodes: Default::default(), + documents: Default::default(), + }) +} + +fn compile_location<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + input: &SnapshotInput, +) -> PyResult { + status(py, "Discovering location project...")?; + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Discovered, + )?; + status(py, "Location project discovered. Compiling snapshot...")?; + let options = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[("selected_target", arguments.value("selected_target")?)], + )?; + let compiled = compiled_phase((&discovered, &adapter), &options, &py.None().into_bound(py))?; + let project = field(&compiled, "project")?; + let view = project.cast::()?.borrow(); + let snapshot = build_orchestration_snapshot( + view, + &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, + project_dir.extract()?, + None, + )?; + status(py, "Location snapshot compiled.")?; + Ok(snapshot) +} + +/// One publication is also the isolated compiler boundary a reloading location can invoke. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_location_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + py, + "run_location_command", + options, + ( + &[ + "project_dir", + "location_id", + "selected_target", + "generation", + "server_url", + ], + &["image_id"], + ), + )?; + let publication = (|| { + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", + ) + })?; + if bearer.trim().is_empty() { + return Err(PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must not be blank", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, &arguments, &directory)?; + let snapshot = compile_location(py, &arguments, &input)?; + status(py, "Publishing location snapshot...")?; + let record = snapshot.publication( + py, + &arguments.value("server_url")?.extract::()?, + &bearer, + )?.map_err(|error| { + let message = if matches!(error, sqlbuild_orchestration::api::errors::ClientError::Transport(_)) { + format!("Publication outcome is unknown; generation {} may already be published. Check the current snapshot before retrying: {error}", input.generation) + } else { + error.to_string() + }; + PyValueError::new_err(message) + })?; + let output = serde_json::to_string(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { + PyValueError::new_err(format!( + "Snapshot {} was published, but result delivery failed: {error}", + record.snapshot_id + )) + })?; + status(py, "Location snapshot published.")?; + Ok(0) + })(); + if publication.is_err() { + status(py, "Location command failed.")?; + } + publication +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_location_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs new file mode 100644 index 0000000000..d5ef1bbf1b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -0,0 +1 @@ +pub(crate) mod command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 457ca748d9..732a57220e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -59,6 +59,7 @@ pub(crate) mod janitor; pub(crate) mod lifecycle_events; pub(crate) mod lineage; pub(crate) mod load; +pub(crate) mod location_command; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod microbatches; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index c2904c48dc..796bb304d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -65,6 +65,14 @@ pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult, stream: &str, text: &str) -> PyResult<()> { + let stream = py.import("sys")?.getattr(stream)?; + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + /// `log_debug_event(logger=logging.getLogger(logger), message=message)`. pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { let kwargs: Bound<'_, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d8e3df098d..6a1c04bac6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -982,6 +982,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::new("run_refactor", FieldDefault::None), FieldSpec::new("run_recall", FieldDefault::None), FieldSpec::new("run_review", FieldDefault::None), + FieldSpec::new("run_location", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1560,6 +1561,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("DIFF", "diff"), ("REVIEW", "review"), ("DEBUG", "debug"), + ("LOCATION", "location"), ("LINEAGE", "lineage"), ("QUERY", "query"), ("COST", "cost"), diff --git a/fensu.toml b/fensu.toml index 191fa4912d..104f5bec85 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2654,6 +2654,7 @@ module_layers = [ "lineage/lineage_command=commands", "lineage/lineage_facts=project", "load=commands", + "location_command=commands", "macro_bridge=front", "manifest=compile", "microbatches=plan", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 27199c0f53..b4579905de 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1769,6 +1769,15 @@ def build_orchestration_snapshot( *, declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... +def run_location_command( + *, + project_dir: Path, + location_id: str, + selected_target: str, + generation: int, + server_url: str, + image_id: str | None = None, +) -> int: ... class NativeProjectView: @property @@ -7302,6 +7311,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None run_recall: Callable[..., int] | None run_review: Callable[..., int] | None + run_location: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -7338,6 +7348,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None = None, run_recall: Callable[..., int] | None = None, run_review: Callable[..., int] | None = None, + run_location: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -8053,6 +8064,7 @@ class CliCommand(StrEnum): DIFF = "diff" REVIEW = "review" DEBUG = "debug" + LOCATION = "location" LINEAGE = "lineage" QUERY = "query" COST = "cost" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 27b44181af..7639ec285f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -13,6 +13,7 @@ from collections.abc import Iterator from contextlib import contextmanager from datetime import UTC, datetime, timedelta +from io import StringIO from pathlib import Path from typing import Any from urllib.parse import quote @@ -25,6 +26,13 @@ ) +class RejectedPublicationOutput(StringIO): + """A caller loses only the result stream, after the real API accepts a publication.""" + + def write(self, text: str) -> int: + raise OSError("synthetic publication result delivery failure") + + def api_address(process: subprocess.Popen[bytes], log: Path) -> str: """Wait for bounded native startup, never publishing an unready address.""" deadline: float = time.monotonic() + 10 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 90d620f024..31caf9be2d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -1,18 +1,23 @@ """Real compile, native transport, authenticated server and durable publication.""" import json +import socket +import sqlite3 +import subprocess from pathlib import Path from typing import Any from urllib.parse import quote from urllib.request import Request, urlopen import pytest +from _pytest.capture import CaptureResult from sqlbuild._native import ( NativeOrchestrationSnapshot, build_compiled_project, build_orchestration_snapshot, discover_project_inputs, + run_cli, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView @@ -24,6 +29,7 @@ SnapshotPublicationTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + RejectedPublicationOutput, current_snapshot, snapshot_api, ) @@ -120,3 +126,151 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert triggers[0]["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" assert triggers[0]["definition_json"]["default_status"] == "paused" assert triggers[0]["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="configured development target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="configured target with encoded path characters", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolated_from_store_and_failed_edits_keep_last_good( + test_case: SnapshotPublicationTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job\n" + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ), + } + write_files(project_dir=project_dir, files=files) + _ = subprocess.run(["git", "init", "-q", str(project_dir)], check=True, capture_output=True) + _ = subprocess.run( + [ + "git", + "-C", + str(project_dir), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Initial orders", + ], + check=True, + capture_output=True, + ) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + arguments: list[str] = [ + "--no-color", + "--project-dir", + str(project_dir), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + "--image-id", + "sha256:synthetic-orders-image", + ] + assert run_cli([*arguments, "--generation", "1"]) == 0 + first_output: CaptureResult[str] = capsys.readouterr() + first: dict[str, Any] = json.loads(first_output.out) + assert first["generation"] == 1 + assert "Location snapshot published." in first_output.err + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["documents"]["jobs"]["daily_orders"]["selection"] == ["model:orders"] + assert not (project_dir / ".sqlbuild/control.sqlite3").exists() + assert not (project_dir / "warehouse.duckdb").exists() + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as connection: + metadata: tuple[str, int, str, str] | None = connection.execute( + "SELECT git_revision, git_dirty, git_author, image_id FROM sqb_snapshots WHERE snapshot_id=?", + (first["snapshot_id"],), + ).fetchone() + assert metadata is not None + assert len(metadata[0]) == 40 + assert metadata[1:] == (1, "Orders Contributor", "sha256:synthetic-orders-image") + + staging: Path = project_dir / "models/stg_orders.sql" + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(11 AS INTEGER)") + ) + assert run_cli([*arguments, "--generation", "2"]) == 0 + second_output: CaptureResult[str] = capsys.readouterr() + second: dict[str, Any] = json.loads(second_output.out) + assert second["generation"] == test_case.expected_generation + updated: dict[str, Any] = current_snapshot(api, test_case.target) + assert {change["subject_key"] for change in updated["changes"]} == {"model:stg_orders"} + assert "CAST(11 AS INTEGER)" in updated["nodes"]["model:stg_orders"]["raw_sql"] + + _ = staging.write_text("SELECT FROM\n") + assert run_cli([*arguments, "--generation", "3"]) != 0 + failed_output: CaptureResult[str] = capsys.readouterr() + assert not failed_output.out + assert "Location command failed." in failed_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + _ = staging.write_text(files["models/stg_orders.sql"]) + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", "invalid-credential") + assert run_cli([*arguments, "--generation", "3"]) != 0 + rejected_output: CaptureResult[str] = capsys.readouterr() + assert not rejected_output.out + assert "HTTP 401" in rejected_output.err + assert api.bearer not in "".join( + ( + first_output.out, + first_output.err, + second_output.out, + second_output.err, + failed_output.err, + rejected_output.err, + ) + ) + assert current_snapshot(api, test_case.target)["record"] == second + + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + with socket.socket() as unavailable: + unavailable.bind(("127.0.0.1", 0)) + disconnected: list[str] = list(arguments) + disconnected[arguments.index("--server") + 1] = ( + f"http://127.0.0.1:{unavailable.getsockname()[1]}" + ) + assert run_cli([*disconnected, "--generation", "3"]) != 0 + transport_output: CaptureResult[str] = capsys.readouterr() + assert not transport_output.out + assert "generation 3 may already be published" in transport_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + with monkeypatch.context() as result_scope: + result_scope.setattr("sys.stdout", RejectedPublicationOutput()) + assert run_cli([*arguments, "--generation", "3"]) != 0 + delivery_output: CaptureResult[str] = capsys.readouterr() + assert "was published, but result delivery failed" in delivery_output.err + assert current_snapshot(api, test_case.target)["record"]["generation"] == 3 From 13a799fe2f872c078be16604f61969dc929f2d60 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:37:48 +0000 Subject: [PATCH 030/138] feat: supervise isolated code location reloads --- Cargo.lock | 1 + Cargo.toml | 1 + crates/sqlbuild-cli/Cargo.toml | 1 + .../sqlbuild-cli/src/argv/command_table.json | 7 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- .../src/launcher/_helpers/dispatch.rs | 6 + .../src/native_commands/_helpers/execution.rs | 119 ++++++++++- .../src/native_commands/_helpers/paths.rs | 54 +++++ .../src/native_commands/main/_run_location.rs | 91 +++++++++ .../src/native_commands/main/mod.rs | 2 + .../main/parse_native_invocation.rs | 16 +- .../main/run_location_invocation.rs | 48 +++++ .../src/native_commands/models.rs | 10 + .../src/snapshots/main/mod.rs | 1 + .../snapshots/main/next_remote_generation.rs | 36 ++++ .../_helpers/location_command/command.rs | 17 +- .../_helpers/location_command/constants.rs | 2 + .../bindings/_helpers/location_command/mod.rs | 1 + src/sqlbuild/_native.pyi | 1 + .../publication/_test_types.py | 15 ++ .../compiled_project/publication/helpers.py | 55 ++++++ .../publication/test_reload.py | 187 ++++++++++++++++++ 22 files changed, 658 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_location.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py diff --git a/Cargo.lock b/Cargo.lock index ad63bc0ef9..dbef530346 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2498,6 +2498,7 @@ dependencies = [ "reqwest", "rusqlite", "rustix", + "same-file", "serde", "serde_json", "sqlbuild-build-features", diff --git a/Cargo.toml b/Cargo.toml index b28edb4dda..01ca1f28c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,7 @@ chrono-tz = "0.10" croner = "=3.0.1" flate2 = "1" proptest = "1" +same-file = "1" semver = "1" # Pin SQLx's driver crates together: its umbrella also resolves an incompatible SQLite engine. sqlx-core = { version = "=0.8.6", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "uuid", "json"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index fd9d3be80e..d5e621fd80 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -37,6 +37,7 @@ rustix.workspace = true [target.'cfg(windows)'.dependencies] ctrlc.workspace = true +same-file.workspace = true [dev-dependencies] reqwest.workspace = true diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index cd3d7e216a..f63070d273 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -106,8 +106,8 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish"], - "help": "compile the project and publish one immutable snapshot", + "choices": ["publish", "serve"], + "help": "publish one snapshot or watch and reload the project", "group": 0 }, { @@ -139,8 +139,7 @@ "options": ["--generation"], "dest": "location_generation", "type": "int", - "required": true, - "help": "positive generation newer than the last successful publication", + "help": "required for publish; serve derives the next generation from the server", "group": 1 }, { diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e224c4f9f4..2f6ce26c7f 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,13 +515,14 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const LOCATION_ARGUMENTS: [Field; 6] = [ +const LOCATION_ARGUMENTS: [Field; 7] = [ field("project_dir", Source::EffectiveProjectDir), SELECTED_TARGET, same("location_id"), attr("server_url", "location_server_url"), attr("generation", "location_generation"), attr("image_id", "location_image_id"), + attr("action", "location_command"), ]; pub(crate) const LOCATION_ROUTE: Route = routed( "run_location", diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index 441901c2b6..ff3bfcc8ea 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -12,6 +12,7 @@ use crate::launcher::constants::{ }; use crate::native_commands::main::parse_native_invocation::parse_native_invocation; use crate::native_commands::main::run_control_invocation::run_control_invocation; +use crate::native_commands::main::run_location_invocation::run_location_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; @@ -55,6 +56,11 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - report_missing_python_cli(&program); return HANDOFF_FAILURE_EXIT_CODE; }; + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_location_invocation(namespace, &script) + { + return code; + } let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); if let Some((path, key)) = &record diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 0a84181be4..66135cf49c 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -2,37 +2,144 @@ use std::io; use std::path::{Path, PathBuf}; +use std::process::Stdio; use std::time::SystemTime; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; +use tokio::process::{Child, Command}; use tokio::sync::watch; use tokio::task::JoinSet; -use crate::argv::models::{Namespace, Stream}; +use crate::argv::models::{Namespace, Stream, Value}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; -use crate::native_commands::_helpers::decision::control_database; +use crate::native_commands::_helpers::decision::{control_database, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::project_location; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; -use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; +use crate::native_commands::_helpers::terminal::{ + finish_lost_stdout, terminal_columns, write_output, +}; +use crate::native_commands::constants::{ + COLUMNS_VARIABLE, FALLBACK_COLUMNS, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, +}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationServe, LostStdout, NativeCommand, NativeInvocation, + ProjectLocation, }; +pub(crate) fn location_options(namespace: &Namespace) -> Result { + if namespace + .get("location_generation") + .is_some_and(|value| *value != Value::None) + { + return Err(ControlError::Invalid( + "serve derives --generation from the server; do not supply it".into(), + )); + } + let project_dir = text(namespace, "project_dir") + .map_or_else(std::env::current_dir, |directory| { + std::fs::canonicalize(directory) + })?; + if !project_dir.is_dir() { + return Err(ControlError::Invalid("project directory must exist".into())); + } + let location_id = text(namespace, "location_id") + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Invalid("location ID must be a UUID".into()))?; + let target = text(namespace, "target").unwrap_or_default(); + if target.trim().is_empty() { + return Err(ControlError::Invalid("target must not be blank".into())); + } + Ok(LocationServe { + project_dir, + server_url: text(namespace, "location_server_url").unwrap_or_default(), + location_id, + target, + image_id: text(namespace, "location_image_id"), + }) +} + +/// Reconcile from the server after restarts or lost acknowledgements; transient outages retry. +pub(crate) async fn location_generation( + options: &LocationServe, + bearer: &str, +) -> Result, ControlError> { + match next_remote_generation( + &options.server_url, + bearer, + options.location_id, + &options.target, + ) + .await + { + Ok(generation) => Ok(Some(generation)), + Err(error) => { + if matches!(&error, ClientError::Configuration(_)) + || matches!(&error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location generation lookup failed; retaining last good snapshot and retrying: {error}\n" + ), + )])); + Ok(None) + } + } +} + +/// The installed interpreter owns user imports; the supervisor never imports Python. +pub(crate) fn publish_child( + options: &LocationServe, + script: &Path, + generation: i64, +) -> io::Result { + let mut command = Command::new(script); + command + .arg("--project-dir") + .arg(&options.project_dir) + .args([ + "location", + "publish", + "--server", + &options.server_url, + "--location-id", + ]) + .arg(options.location_id.to_string()) + .args(["--target", &options.target, "--generation"]) + .arg(generation.to_string()); + if let Some(image) = &options.image_id { + command.args(["--image-id", image]); + } + if terminal_columns().is_err() { + command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); + } + command + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() +} + /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. pub(crate) async fn open_control_store( namespace: &Namespace, @@ -48,7 +155,7 @@ pub(crate) async fn open_control_store( Ok(store) } -async fn shutdown_signal() -> Result<(), ControlError> { +pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { #[cfg(unix)] { let mut terminate = diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 302e8a5a66..565ecf7f95 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -2,10 +2,64 @@ use std::path::{Component, Path, PathBuf}; +use sqlbuild_cache::digest::constants::{ + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, +}; +use sqlbuild_cache::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::ProjectLocation; +use crate::replay::main::output_file_identities::output_file_identities; + +/// Observe the same project inputs as compiler reuse, excluding redirected command output. +pub(crate) fn location_rules() -> SnapshotRules { + SnapshotRules { + excluded: EXCLUDED_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + excluded_root: EXCLUDED_ROOT_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + presence_suffixes: PRESENCE_ONLY_FILE_SUFFIXES + .iter() + .map(ToString::to_string) + .collect(), + output_files: output_file_identities(), + } +} + +/// Git commits matter to publication metadata even when no working-tree file changes. +pub(crate) fn location_inputs( + root: &Path, + rules: &SnapshotRules, +) -> (Vec, Option) { + let mut stamps = snapshot_project_files(root, rules); + #[cfg(windows)] + { + let output: Vec = + [same_file::Handle::stdout(), same_file::Handle::stderr()] + .into_iter() + .flatten() + .filter(|handle| { + handle + .as_file() + .metadata() + .is_ok_and(|metadata| metadata.is_file()) + }) + .collect(); + stamps.retain(|stamp| { + same_file::Handle::from_path(root.join(&stamp.relative_path)) + .map_or(true, |handle| !output.contains(&handle)) + }); + } + stamps.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + (stamps, head_commit(root)) +} /// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. pub(crate) fn pathlib_path(text: &Path) -> PathBuf { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs new file mode 100644 index 0000000000..017c4fd008 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -0,0 +1,91 @@ +//! Coalesce source edits and retain the last good publication across isolated compiler failures. + +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::time::{Instant, MissedTickBehavior}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::{ + location_generation, publish_child, shutdown_signal, +}; +use crate::native_commands::_helpers::paths::{location_inputs, location_rules}; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::models::LocationServe; + +pub(super) async fn run_location( + options: LocationServe, + script: &Path, + bearer: &str, +) -> Result<(), ControlError> { + let rules = location_rules(); + let mut observed = location_inputs(&options.project_dir, &rules); + let mut attempted = None; + let mut changed_at = Instant::now(); + let mut retry_at = None; + let mut clock = tokio::time::interval(Duration::from_millis(250)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + let stopped = shutdown_signal(); + tokio::pin!(stopped); + drop(write_output(&[( + Stream::Stderr, + "Starting code location; watching project inputs...\n", + )])); + loop { + tokio::select! { + result = &mut stopped => return result, + _ = clock.tick() => {}, + } + let current = location_inputs(&options.project_dir, &rules); + if observed != current { + observed = current; + changed_at = Instant::now(); + } + if attempted.as_ref() == Some(&observed) + && retry_at.is_none_or(|retry| Instant::now() < retry) + { + continue; + } + if attempted.is_some() && changed_at.elapsed() < Duration::from_millis(250) { + continue; + } + attempted = Some(observed.clone()); + let generation = tokio::select! { + result = &mut stopped => return result, + generation = location_generation(&options, bearer) => generation?, + }; + let generation = match generation { + Some(generation) => generation, + None => { + retry_at = Some(Instant::now() + Duration::from_secs(5)); + continue; + } + }; + drop(write_output(&[( + Stream::Stderr, + &format!("Compiling location generation {generation} in an isolated interpreter...\n"), + )])); + let mut child = publish_child(&options, script, generation)?; + let result = tokio::select! { + result = &mut stopped => { + child.kill().await?; + return result; + }, + result = child.wait() => result?, + }; + if result.success() { + retry_at = None; + drop(write_output(&[( + Stream::Stderr, + "Code location ready; watching for changes.\n", + )])); + } else { + retry_at = Some(Instant::now() + Duration::from_secs(30)); + drop(write_output(&[( + Stream::Stderr, + "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", + )])); + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index e93969bdc4..ac5f9232e5 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -2,6 +2,8 @@ pub(super) mod _cost_report_options; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; +pub(super) mod _run_location; pub(crate) mod parse_native_invocation; pub(crate) mod run_control_invocation; +pub(crate) mod run_location_invocation; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs index 707115b4f9..8c2056d976 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -5,11 +5,11 @@ use std::ffi::OsString; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; -use crate::native_commands::_helpers::decision::review_json_failure; +use crate::native_commands::_helpers::decision::{review_json_failure, text}; use crate::native_commands::_helpers::terminal::{ finish_rejected_parse, stdout_supports_color, terminal_columns, }; -use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::constants::{FALLBACK_COLUMNS, NO_COLOR_OPTION}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::NativeParse; @@ -21,13 +21,23 @@ pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result Option { + if text(namespace, "command").as_deref() != Some("location") + || text(namespace, "location_command").as_deref() != Some("serve") + { + return None; + } + let result = (|| { + let options = location_options(namespace)?; + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + std::io::Error::other( + "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", + ) + })?; + if bearer.trim().is_empty() { + return Err(std::io::Error::other("SQLBUILD_LOCATION_TOKEN must not be blank").into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(run_location(options, script, &bearer)) + })(); + Some(match result { + Ok(()) => { + drop(write_output(&[( + Stream::Stderr, + "Code location stopped successfully\n", + )])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Code location failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 7d7cbc3df5..7be0c55c84 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -126,6 +126,16 @@ pub(crate) struct ProjectLocation { pub(crate) resolved: PathBuf, } +/// A long-running location owns no control-store credentials or executable user objects. +#[derive(Debug)] +pub(crate) struct LocationServe { + pub(crate) project_dir: PathBuf, + pub(crate) server_url: String, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) image_id: Option, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 8411aaf37d..56f1b75755 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,4 +1,5 @@ pub mod current_snapshot; +pub mod next_remote_generation; pub mod publish_remote_snapshot; pub mod publish_snapshot; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs new file mode 100644 index 0000000000..f92dabe4fa --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs @@ -0,0 +1,36 @@ +//! Reconcile publication generations from the authenticated server, including lost acknowledgements. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::CurrentSnapshot; + +pub async fn next_remote_generation( + server_url: &str, + bearer: &str, + location_id: Uuid, + target: &str, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &location_id.to_string(), + "targets", + target, + "snapshots", + ])?; + match client_response::(connection.client.get(url)).await { + Ok(current) => current + .record + .generation + .checked_add(1) + .ok_or(ClientError::Configuration( + "snapshot generation is exhausted", + )), + Err(ClientError::Rejected(status)) if status == reqwest::StatusCode::NOT_FOUND => Ok(1), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 001f7211e1..1ec1b8a2ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ NativeOrchestrationSnapshot, build_orchestration_snapshot, }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -39,7 +40,10 @@ fn metadata( .parse() .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; let target: String = arguments.value("selected_target")?.extract()?; - let generation: i64 = arguments.value("generation")?.extract()?; + let generation = arguments + .value("generation")? + .extract::>()? + .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; if target.trim().is_empty() || generation < 1 { return Err(PyValueError::new_err( "target must not be blank and generation must be positive", @@ -145,10 +149,19 @@ pub(crate) fn run_location_command<'py>( "generation", "server_url", ], - &["image_id"], + &["image_id", "action"], ), )?; let publication = (|| { + if arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action != PUBLISH_ACTION) + { + return Err(PyValueError::new_err( + "location serve must run through the native sqb executable", + )); + } let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { PyValueError::new_err( "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs new file mode 100644 index 0000000000..dc0a1d6e2d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -0,0 +1,2 @@ +/// Location actions accepted by the interpreter-side one-shot publisher. +pub(crate) const PUBLISH_ACTION: &str = "publish"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index d5ef1bbf1b..8aa0ebb43e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1 +1,2 @@ pub(crate) mod command; +mod constants; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b4579905de..b235862344 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1777,6 +1777,7 @@ def run_location_command( generation: int, server_url: str, image_id: str | None = None, + action: str = "publish", ) -> int: ... class NativeProjectView: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 63a996d791..be66d72ee4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -31,3 +31,18 @@ class SnapshotWorkerTestCase: original_sql: str replacement_sql: str expected_error: str + + +@dataclass(frozen=True) +class LocationReloadTestCase: + description: str + target: str + expected_generation: int + expected_stop_code: int + + +@dataclass(frozen=True) +class LocationShutdownTestCase: + description: str + signal: int + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 7639ec285f..994e90c8cc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -16,6 +16,7 @@ from io import StringIO from pathlib import Path from typing import Any +from urllib.error import HTTPError from urllib.parse import quote from urllib.request import Request, urlopen from uuid import uuid4 @@ -33,6 +34,60 @@ def write(self, text: str) -> int: raise OSError("synthetic publication result delivery failure") +@contextmanager +def location_process( + arguments: list[str], log: Path, publications: Path +) -> Iterator[subprocess.Popen[bytes]]: + """Own and reap the supervisor even when an assertion interrupts a reload.""" + with log.open("ab") as stderr, publications.open("ab") as stdout: + process: subprocess.Popen[bytes] = subprocess.Popen(arguments, stderr=stderr, stdout=stdout) + try: + yield process + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def wait_for_generation( + api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + current: dict[str, Any] = {"record": {"generation": 0}} + while current["record"]["generation"] < generation: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, ( + f"Generation {generation} was not published: {log.read_text()}" + ) + try: + current = current_snapshot(api, target) + except HTTPError as error: + assert error.code == 404 + time.sleep(0.02) + assert current["record"]["generation"] == generation, log.read_text() + return current + + +def wait_for_log(process: subprocess.Popen[bytes], log: Path, text: str) -> None: + deadline: float = time.monotonic() + 15 + while text not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + return int(pidfile.read_text()) + + def api_address(process: subprocess.Popen[bytes], log: Path) -> str: """Wait for bounded native startup, never publishing an unready address.""" deadline: float = time.monotonic() + 10 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py new file mode 100644 index 0000000000..d2cbbe1f13 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -0,0 +1,187 @@ +"""A native supervisor reloads real project interpreters through the authenticated HTTP API.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationReloadTestCase, + LocationShutdownTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_log, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationReloadTestCase("ordinary target", "development", 5, int(sys.platform == "win32")), + LocationReloadTestCase("encoded target", "preview #1", 5, int(sys.platform == "win32")), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_inputs_change_or_restart_then_generations_reload_without_losing_last_good( + test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + declaration: str = ( + "from sqlbuild.orchestration import job\n" + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ) + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": declaration, + } + write_files(project_dir=project, files=files) + _ = subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # A presentation-only width fallback must not bypass the native-only supervisor. + monkeypatch.setenv("COLUMNS", "999999999999999999999999999999") + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + ] + log: Path = project / "location.log" + publications: Path = project / "publications.jsonl" + with location_process(arguments, log, publications) as process: + first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + staging: Path = project / "models/stg_orders.sql" + for value in (11, 12, 13): + _ = staging.write_text( + files["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", f"CAST({value} AS INTEGER)" + ) + ) + second: dict[str, Any] = wait_for_generation(api, test_case.target, 2, process, log) + assert second["record"]["snapshot_id"] != first["record"]["snapshot_id"] + assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] + assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} + + _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + assert current_snapshot(api, test_case.target)["record"] == second["record"] + _ = (project / "orchestration/orders.py").write_text( + declaration.replace("Build orders.", "Refresh order totals.") + ) + third: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + assert ( + third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." + ) + _ = subprocess.run( + [ + "git", + "-C", + str(project), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Record orders", + ], + check=True, + capture_output=True, + ) + committed: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + assert committed["nodes"] == third["nodes"] + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + with location_process(arguments, log, publications) as process: + restarted: dict[str, Any] = wait_for_generation( + api, test_case.target, test_case.expected_generation, process, log + ) + assert restarted["nodes"] == third["nodes"] + # Redirected output and compiler cache writes must not feed the input watcher. + time.sleep(1) + assert current_snapshot(api, test_case.target)["record"] == restarted["record"] + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + records: list[dict[str, Any]] = [ + json.loads(line) for line in publications.read_text().splitlines() + ] + assert [record["generation"] for record in records] == [1, 2, 3, 4, 5] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + assert not (project / "warehouse.duckdb").exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") +@pytest.mark.parametrize( + "test_case", + [ + LocationShutdownTestCase("terminate compiling interpreter", signal.SIGTERM, 0), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0), + ], + ids=lambda case: case.description, +) +def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( + test_case: LocationShutdownTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + child_pid: Path = tmp_path / "interpreter.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/slow.py": ( + "import os, time\nfrom pathlib import Path\n" + f"Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + "time.sleep(60)\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + interpreter: int = wait_for_interpreter(process, child_pid, log) + process.send_signal(test_case.signal) + assert process.wait(timeout=5) == test_case.expected_exit_code + with pytest.raises(ProcessLookupError): + os.kill(interpreter, 0) From bf359c1d5a01ac7f5e0739243581f95780f9331e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:18:25 +0000 Subject: [PATCH 031/138] feat: report scoped code location health --- .../src/native_commands/_helpers/execution.rs | 104 ++++++++- .../src/native_commands/_helpers/paths.rs | 94 +++++++- .../src/native_commands/main/_run_location.rs | 106 +++++++--- .../src/native_commands/models.rs | 12 ++ .../debug_report/main/format_debug_json.rs | 11 + .../sqlbuild-core/src/debug_report/models.rs | 2 + .../src/debug_report/tests/helpers.rs | 1 + .../debug_report/tests/test_debug_report.rs | 2 +- .../src/api/_helpers/location_health.rs | 45 ++++ .../src/api/_helpers/mod.rs | 1 + .../src/api/main/serve_api.rs | 6 +- .../sqlbuild-orchestration/src/api/models.rs | 4 +- .../platform/_helpers/location_heartbeat.rs | 50 +++++ .../src/platform/_helpers/mod.rs | 1 + .../src/platform/main/location_health.rs | 63 ++++++ .../src/platform/main/mod.rs | 3 + .../platform/main/report_location_health.rs | 104 +++++++++ .../main/report_remote_location_health.rs | 21 ++ .../src/platform/models.rs | 25 +++ .../src/store/_helpers/migration.rs | 11 +- .../src/store/location_instances.sql | 9 + .../src/store/main/migrate_store.rs | 20 +- crates/sqlbuild-orchestration/tests/api.rs | 2 + .../tests/api/test_location_health.rs | 200 ++++++++++++++++++ .../tests/api/test_types.rs | 6 + .../tests/snapshots/helpers.rs | 21 +- .../_helpers/command_runtime/connection.rs | 65 +++++- .../_helpers/debug_command/run_debug.rs | 5 +- .../publication/_test_types.py | 12 ++ .../compiled_project/publication/helpers.py | 32 +++ .../publication/test_reload.py | 154 +++++++++++++- 31 files changed, 1119 insertions(+), 73 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs create mode 100644 crates/sqlbuild-orchestration/src/store/location_instances.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_location_health.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 66135cf49c..30241401e9 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1,19 +1,25 @@ //! Running a Python-free command: its checks, then its output with the observed services. +use std::future::Future; use std::io; use std::path::{Path, PathBuf}; use std::process::Stdio; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; +use tokio::io::AsyncReadExt; use tokio::process::{Child, Command}; use tokio::sync::watch; use tokio::task::JoinSet; +use tokio::time::MissedTickBehavior; +use uuid::Uuid; use crate::argv::models::{Namespace, Stream, Value}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; @@ -25,7 +31,7 @@ use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; use crate::native_commands::_helpers::decision::{control_database, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; -use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::paths::{location_database_files, project_location}; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; @@ -42,6 +48,43 @@ use crate::native_commands::models::{ ProjectLocation, }; +/// Health IO remains native even while a user-code child holds its interpreter's GIL. +pub(crate) async fn report_location_health( + options: &LocationServe, + bearer: &str, + session_id: Uuid, + health: LocationHealth, +) { + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: options.target.clone(), + health, + }; + if let Err(error) = + report_remote_location_health((&options.server_url, bearer), options.location_id, &report) + .await + { + drop(write_output(&[( + Stream::Stderr, + &format!("Location health report failed: {error}\n"), + )])); + } +} + +pub(crate) async fn location_heartbeats( + options: &LocationServe, + bearer: &str, + session_id: Uuid, +) -> Result<(), ControlError> { + let mut clock = tokio::time::interval(Duration::from_secs(30)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + clock.tick().await; + report_location_health(options, bearer, session_id, LocationHealth::Heartbeat {}).await; + } +} + pub(crate) fn location_options(namespace: &Namespace) -> Result { if namespace .get("location_generation") @@ -112,10 +155,8 @@ pub(crate) fn publish_child( script: &Path, generation: i64, ) -> io::Result { - let mut command = Command::new(script); + let mut command = location_child(options, script); command - .arg("--project-dir") - .arg(&options.project_dir) .args([ "location", "publish", @@ -129,15 +170,62 @@ pub(crate) fn publish_child( if let Some(image) = &options.image_id { command.args(["--image-id", image]); } + command.stdout(Stdio::inherit()).spawn() +} + +/// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. +pub(crate) async fn check_location_connection( + options: &LocationServe, + script: &Path, + stopped: impl Future>, +) -> Result)>, ControlError> { + let mut child = location_child(options, script) + .args(["debug", "--target", &options.target, "--json"]) + .stdout(Stdio::piped()) + .spawn()?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("location debug output is unavailable".into()))?; + let mut output: Vec = Vec::new(); + tokio::pin!(stopped); + let connected = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = async { tokio::try_join!(child.wait(), stdout.read_to_end(&mut output)) } => result?.0.success(), + }; + let files: Vec = match serde_json::from_slice::(&output) + .and_then(|report| serde_json::from_value(report["local_files"].clone())) + { + Ok(files) => files, + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location connection check did not produce a valid report; retaining the published snapshot.\n", + )])); + return Ok(Some((false, Vec::new()))); + } + }; + Ok(Some(( + connected, + location_database_files(&options.project_dir, files), + ))) +} + +fn location_child(options: &LocationServe, script: &Path) -> Command { + let mut command = Command::new(script); + command.arg("--project-dir").arg(&options.project_dir); if terminal_columns().is_err() { command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); } command .stdin(Stdio::null()) - .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() + .kill_on_drop(true); + command } /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 565ecf7f95..e51e498caa 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -1,6 +1,7 @@ //! `pathlib.Path` construction and `Path.resolve()` over native paths. use std::path::{Component, Path, PathBuf}; +use std::time::Duration; use sqlbuild_cache::digest::constants::{ EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, @@ -9,13 +10,48 @@ use sqlbuild_cache::project_snapshot::main::snapshot_project_files::snapshot_pro use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; +use tokio::time::Instant; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::ProjectLocation; +use crate::native_commands::models::{LocationWatch, ProjectLocation}; use crate::replay::main::output_file_identities::output_file_identities; /// Observe the same project inputs as compiler reuse, excluding redirected command output. -pub(crate) fn location_rules() -> SnapshotRules { +pub(crate) fn location_watch(root: &Path) -> LocationWatch { + let rules = location_rules(); + LocationWatch { + observed: location_inputs(root, &rules, &[]), + rules, + local_files: Vec::new(), + attempted: None, + changed_at: tokio::time::Instant::now(), + retry_at: None, + } +} + +/// Coalesce source edits and retry deadlines without replacing the retained attempted input. +pub(crate) fn observe_location_inputs( + root: &Path, + mut watch: LocationWatch, +) -> (LocationWatch, bool) { + let current = location_inputs(root, &watch.rules, &watch.local_files); + if watch.observed != current { + watch.observed = current; + watch.changed_at = Instant::now(); + } + if watch.attempted.as_ref() == Some(&watch.observed) + && watch.retry_at.is_none_or(|retry| Instant::now() < retry) + { + return (watch, false); + } + if watch.attempted.is_some() && watch.changed_at.elapsed() < Duration::from_millis(250) { + return (watch, false); + } + watch.attempted = Some(watch.observed.clone()); + (watch, true) +} + +fn location_rules() -> SnapshotRules { SnapshotRules { excluded: EXCLUDED_DIRECTORIES .iter() @@ -34,11 +70,19 @@ pub(crate) fn location_rules() -> SnapshotRules { } /// Git commits matter to publication metadata even when no working-tree file changes. -pub(crate) fn location_inputs( +fn location_inputs( root: &Path, rules: &SnapshotRules, + local_files: &[PathBuf], ) -> (Vec, Option) { let mut stamps = snapshot_project_files(root, rules); + stamps.retain(|stamp| { + !rules + .presence_suffixes + .iter() + .any(|suffix| stamp.relative_path.ends_with(suffix)) + }); + stamps = exclude_location_files(stamps, root, local_files); #[cfg(windows)] { let output: Vec = @@ -61,6 +105,50 @@ pub(crate) fn location_inputs( (stamps, head_commit(root)) } +/// Apply resolved exclusions to retained observations, never a fresh snapshot of possible source edits. +pub(crate) fn exclude_location_databases( + root: &Path, + files: Vec, + mut watch: LocationWatch, +) -> LocationWatch { + watch.local_files.extend(files); + watch.local_files.sort(); + watch.local_files.dedup(); + watch.observed.0 = exclude_location_files(watch.observed.0, root, &watch.local_files); + watch.attempted = watch.attempted.map(|(stamps, commit)| { + ( + exclude_location_files(stamps, root, &watch.local_files), + commit, + ) + }); + watch +} + +fn exclude_location_files( + mut stamps: Vec, + root: &Path, + local_files: &[PathBuf], +) -> Vec { + stamps.retain(|stamp| !local_files.contains(&root.join(&stamp.relative_path))); + stamps +} + +pub(crate) fn location_database_files(root: &Path, files: Vec) -> Vec { + let mut paths: Vec = Vec::new(); + for file in files { + let authored = pathlib_path(&root.join(file)); + let resolved = project_real_path(&authored); + for path in [authored, resolved] { + for suffix in ["", ".wal", "-wal", "-shm", "-journal"] { + let mut file = path.as_os_str().to_os_string(); + file.push(suffix); + paths.push(PathBuf::from(file)); + } + } + } + paths +} + /// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. pub(crate) fn pathlib_path(text: &Path) -> PathBuf { let path: PathBuf = text diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 017c4fd008..b7bff31ea3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -1,16 +1,22 @@ //! Coalesce source edits and retain the last good publication across isolated compiler failures. -use std::path::Path; +use std::future::Future; +use std::path::{Path, PathBuf}; use std::time::Duration; +use sqlbuild_orchestration::platform::models::LocationHealth; use sqlbuild_orchestration::store::errors::ControlError; use tokio::time::{Instant, MissedTickBehavior}; +use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - location_generation, publish_child, shutdown_signal, + check_location_connection, location_generation, location_heartbeats, publish_child, + report_location_health, shutdown_signal, +}; +use crate::native_commands::_helpers::paths::{ + exclude_location_databases, location_watch, observe_location_inputs, }; -use crate::native_commands::_helpers::paths::{location_inputs, location_rules}; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::models::LocationServe; @@ -19,11 +25,20 @@ pub(super) async fn run_location( script: &Path, bearer: &str, ) -> Result<(), ControlError> { - let rules = location_rules(); - let mut observed = location_inputs(&options.project_dir, &rules); - let mut attempted = None; - let mut changed_at = Instant::now(); - let mut retry_at = None; + let session_id = Uuid::now_v7(); + tokio::select! { + result = watch_location(&options, script, bearer, session_id) => result, + result = location_heartbeats(&options, bearer, session_id) => result, + } +} + +async fn watch_location( + options: &LocationServe, + script: &Path, + bearer: &str, + session_id: Uuid, +) -> Result<(), ControlError> { + let mut watch = location_watch(&options.project_dir); let mut clock = tokio::time::interval(Duration::from_millis(250)); clock.set_missed_tick_behavior(MissedTickBehavior::Skip); let stopped = shutdown_signal(); @@ -37,28 +52,19 @@ pub(super) async fn run_location( result = &mut stopped => return result, _ = clock.tick() => {}, } - let current = location_inputs(&options.project_dir, &rules); - if observed != current { - observed = current; - changed_at = Instant::now(); - } - if attempted.as_ref() == Some(&observed) - && retry_at.is_none_or(|retry| Instant::now() < retry) - { - continue; - } - if attempted.is_some() && changed_at.elapsed() < Duration::from_millis(250) { + let ready; + (watch, ready) = observe_location_inputs(&options.project_dir, watch); + if !ready { continue; } - attempted = Some(observed.clone()); let generation = tokio::select! { result = &mut stopped => return result, - generation = location_generation(&options, bearer) => generation?, + generation = location_generation(options, bearer) => generation?, }; let generation = match generation { Some(generation) => generation, None => { - retry_at = Some(Instant::now() + Duration::from_secs(5)); + watch.retry_at = Some(Instant::now() + Duration::from_secs(5)); continue; } }; @@ -66,7 +72,7 @@ pub(super) async fn run_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let mut child = publish_child(&options, script, generation)?; + let mut child = publish_child(options, script, generation)?; let result = tokio::select! { result = &mut stopped => { child.kill().await?; @@ -75,13 +81,24 @@ pub(super) async fn run_location( result = child.wait() => result?, }; if result.success() { - retry_at = None; - drop(write_output(&[( - Stream::Stderr, - "Code location ready; watching for changes.\n", - )])); + watch.retry_at = None; + let Some(files) = check_published_location( + options, + script, + (bearer, session_id, generation), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); } else { - retry_at = Some(Instant::now() + Duration::from_secs(30)); + watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); + tokio::select! { + result = &mut stopped => return result, + () = report_location_health(options, bearer, session_id, LocationHealth::CompileFailed { generation }) => {}, + } drop(write_output(&[( Stream::Stderr, "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", @@ -89,3 +106,34 @@ pub(super) async fn run_location( } } } + +async fn check_published_location( + options: &LocationServe, + script: &Path, + (bearer, session_id, generation): (&str, Uuid, i64), + stopped: impl Future>, +) -> Result>, ControlError> { + drop(write_output(&[( + Stream::Stderr, + "Checking location connection with sqb debug...\n", + )])); + tokio::pin!(stopped); + let Some((connected, files)) = + check_location_connection(options, script, stopped.as_mut()).await? + else { + return Ok(None); + }; + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + if connected { + "Code location ready; connection check passed; watching for changes.\n" + } else { + "Code location published; connection check failed; watching for changes.\n" + }, + )])); + Ok(Some(files)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 7be0c55c84..f3ae56696a 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::runs::models::JobRunRequest; @@ -136,6 +137,17 @@ pub(crate) struct LocationServe { pub(crate) image_id: Option, } +/// Retained source observations survive compilation; warehouse exclusions never replace them. +#[derive(Debug)] +pub(crate) struct LocationWatch { + pub(crate) rules: SnapshotRules, + pub(crate) local_files: Vec, + pub(crate) observed: (Vec, Option), + pub(crate) attempted: Option<(Vec, Option)>, + pub(crate) changed_at: tokio::time::Instant, + pub(crate) retry_at: Option, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs index 04e8d3084f..9eb5e8408f 100644 --- a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -43,6 +43,17 @@ pub fn format_debug_json(report: &DebugReport) -> String { ), ("providers".to_owned(), lines_value(&report.providers)), ("connection".to_owned(), lines_value(&report.connection)), + ( + "local_files".to_owned(), + JsonValue::Array( + report + .local_files + .iter() + .cloned() + .map(JsonValue::String) + .collect(), + ), + ), ]); let options: StdlibJsonOptions = StdlibJsonOptions { sort_keys: true, diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index c0ec5075af..144079f7d9 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -38,6 +38,8 @@ pub struct DebugReport { pub configuration: Vec, pub providers: Vec, pub connection: Vec, + /// Exact local database paths, not the shortened display values in connection lines. + pub local_files: Vec, } impl DebugReport { diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs index 623f516cd6..240efc55d5 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -2,6 +2,7 @@ use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, Debug pub(super) fn sample_report() -> DebugReport { DebugReport { + local_files: vec!["/repo/warehouse.db".into()], runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], configuration: vec![DebugLine::checked( "project file", diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index 7a2122f2d7..5fb74ee026 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -18,7 +18,7 @@ fn given_debug_report_when_formatting_then_matches_python_outputs() { description: "renders diagnostics sections and json sections", report: sample_report(), expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", - expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, expected_color_fragments: &[ "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", "\x1b[1mRuntime:\x1b[0m", diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs new file mode 100644 index 0000000000..60a407233c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs @@ -0,0 +1,45 @@ +//! Scope both observations and reads before exposing any location history. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::location_health::location_health; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealthObservation, LocationHealthReport}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/health", + params(("location_id" = Uuid, Path)), request_body = LocationHealthReport, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(report): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: report_location_health(&store, location_id, &report).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/health", + params(("location_id" = Uuid, Path)), + responses((status = 200, body = Vec), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_health(&store, location_id).await?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index ed6ed88be8..4b3f4f829e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod authentication; pub(crate) mod client; +pub(crate) mod location_health; pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 52c2efba75..0a7ba5c5c2 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -5,7 +5,7 @@ use axum::routing::{get, post}; use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; -use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; @@ -19,6 +19,10 @@ pub async fn serve_api( ) -> Result<(), ControlError> { migrate_store(&store).await?; let protected = Router::new() + .route( + "/api/locations/{location_id}/health", + get(location_health::recent).post(location_health::report), + ) .route( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c2604cc5c5..5e4837fa3b 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,10 +2,12 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( + location_health::report, + location_health::recent, snapshots::publish, snapshots::current, runs::execution, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs new file mode 100644 index 0000000000..a13e3fabb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs @@ -0,0 +1,50 @@ +//! A heartbeat replaces one instance's liveness, never appending an execution fact. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::LocationHealthReport; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn location_heartbeat( + transaction: ControlTransaction, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT instance.location_id AS instance_location, instance.health_json AS report \ + FROM sqb_locations location LEFT JOIN sqb_location_instances instance ON instance.instance_id=$1 \ + WHERE location.location_id=$2 AND location.archived_at IS NULL", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id)]).await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("active location is required".into()))?; + if let Some(previous) = row.get("report").filter(|value| !value.is_null()) { + let previous: LocationHealthReport = serde_json::from_value(previous.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if row.get("instance_location") != Some(&json!(location_id)) + || previous.target != report.target + { + return Err(ControlError::Invalid( + "location instance belongs to another scope".into(), + )); + } + if previous == *report { + return Ok((transaction, false)); + } + if previous.event_id == report.event_id { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_instances (instance_id, location_id, target, connected_at, last_seen_at, health_json) \ + VALUES ($1, $2, $3, $4, $4, $5) ON CONFLICT (instance_id) DO UPDATE SET last_seen_at=$4, health_json=$5", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id), Parameter::Text(report.target.clone()), + Parameter::Time(now), Parameter::Json(json!(report))]).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index b604790017..e32efd2a5f 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod location_heartbeat; pub(crate) mod observations; pub(crate) mod watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs new file mode 100644 index 0000000000..977a61e6c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs @@ -0,0 +1,63 @@ +//! Read bounded recent observations and instance liveness; consumers derive freshness. + +use uuid::Uuid; + +use crate::platform::models::LocationHealthObservation; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn location_health( + store: &ControlStore, + location_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let today = transaction.now.date_naive(); + let yesterday = today - chrono::TimeDelta::days(1); + let (transaction, rows) = query( + transaction, + "SELECT received_at, payload FROM (\ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$1 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) current_day \ + UNION ALL \ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$2 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) previous_day\ + ) observations ORDER BY received_at DESC, invocation_sequence DESC LIMIT 100", + vec![ + Parameter::Text(format!("location/{location_id}/{today}")), + Parameter::Text(format!("location/{location_id}/{yesterday}")), + ], + ) + .await?; + let mut observations: Vec = rows + .into_iter() + .map(|mut row| { + let report = row + .remove("payload") + .and_then(|mut payload| payload.get_mut("report").map(serde_json::Value::take)) + .ok_or_else(|| ControlError::Invalid("location health report is missing".into()))?; + row.insert("report".into(), report); + decode_row(row) + }) + .collect::>()?; + let (transaction, instances) = query( + transaction, + "SELECT last_seen_at AS received_at, health_json AS report FROM sqb_location_instances \ + WHERE location_id=$1 ORDER BY last_seen_at DESC LIMIT 100", + vec![Parameter::Id(location_id)], + ) + .await?; + observations.extend( + instances + .into_iter() + .map(decode_row) + .collect::, _>>()?, + ); + observations.sort_by_key(|observation| std::cmp::Reverse(observation.received_at)); + observations.truncate(100); + commit(transaction).await?; + Ok(observations) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index bb879e253f..a633f3e7ce 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1 +1,4 @@ +pub mod location_health; +pub mod report_location_health; +pub mod report_remote_location_health; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs new file mode 100644 index 0000000000..389d80fa18 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -0,0 +1,104 @@ +//! Append scoped location observations without turning health into mutable lifecycle state. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::_helpers::location_heartbeat::location_heartbeat; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlStore, Parameter, TransactionMode}; + +pub async fn report_location_health( + store: &ControlStore, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + if report.target.trim().is_empty() { + return Err(ControlError::Invalid( + "location health target is invalid".into(), + )); + } + let generation = match report.health { + LocationHealth::Heartbeat {} => None, + LocationHealth::CompileFailed { generation } + | LocationHealth::Connection { generation, .. } => Some(generation), + }; + if generation.is_some_and(|generation| generation < 1) { + return Err(ControlError::Invalid( + "location health generation must be positive".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + if generation.is_none() { + let (transaction, applied) = location_heartbeat(transaction, location_id, report).await?; + commit(transaction).await?; + return Ok(applied); + } + let invocation_id = format!("location/{location_id}/{}", transaction.now.date_naive()); + let event_id = Uuid::new_v5(&location_id, report.event_id.as_bytes()); + let payload = json!({"location_id": location_id, "report": report}); + let (transaction, previous) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("payload") != Some(&payload) { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, location) = query( + transaction, + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ) + .await?; + if location.is_empty() { + return Err(ControlError::Invalid("active location is required".into())); + } + let transaction = if let LocationHealth::Connection { generation, .. } = report.health { + let (transaction, snapshot) = query( + transaction, + "SELECT snapshot_id FROM sqb_snapshots WHERE location_id=$1 AND target=$2 AND generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(report.target.clone()), Parameter::Integer(generation)], + ).await?; + if snapshot.is_empty() { + return Err(ControlError::Invalid( + "connection health requires a published generation".into(), + )); + } + transaction + } else { + transaction + }; + let occurred_at = transaction.now; + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id, + event_type: "location_health", + producer: "location", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "location health conflicts with a durable fact".into(), + )); + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs new file mode 100644 index 0000000000..5963cf20b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs @@ -0,0 +1,21 @@ +//! The code location has only its scoped HTTP credential, never a store handle. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::api::models::MutationResponse; +use crate::platform::models::LocationHealthReport; + +pub async fn report_remote_location_health( + (server_url, bearer): (&str, &str), + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&["api", "locations", &location_id.to_string(), "health"])?; + let response: MutationResponse = + client_response(connection.client.post(url).json(report)).await?; + Ok(response.applied) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 0f3bb0325c..052d9bf775 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -6,6 +6,31 @@ use uuid::Uuid; use crate::declarations::models::TriggerDefinition; +/// Locations report bounded facts, never connection strings or user exception text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationHealth { + Heartbeat {}, + CompileFailed { generation: i64 }, + Connection { generation: i64, healthy: bool }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationHealthReport { + pub event_id: Uuid, + pub session_id: Uuid, + pub target: String, + pub health: LocationHealth, +} + +/// Receipt time belongs to the server; consumers derive freshness rather than storing availability. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationHealthObservation { + pub received_at: DateTime, + pub report: LocationHealthReport, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index a29a62c75f..e2933cdb64 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -7,11 +7,12 @@ use crate::store::models::ControlStore; use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); -pub(crate) fn verify_migrations(rows: &[Row], digest: &str) -> Result { +pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) .map_err(|error| ControlError::Schema(error.to_string()))?; - for row in rows { + for (index, row) in rows.iter().enumerate() { let minimum = row .get("min_server_version") .and_then(|value| value.as_str()) @@ -20,9 +21,11 @@ pub(crate) fn verify_migrations(rows: &[Row], digest: &str) -> Result current - || row.get("checksum").and_then(|value| value.as_str()) != Some(digest) + || row.get("checksum").and_then(|value| value.as_str()) + != digests.get(index).map(String::as_str) + || index >= digests.len() { return Err(ControlError::Schema( "unknown migration, checksum mismatch, or minimum server version not met".into(), diff --git a/crates/sqlbuild-orchestration/src/store/location_instances.sql b/crates/sqlbuild-orchestration/src/store/location_instances.sql new file mode 100644 index 0000000000..04b3e8dfd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/location_instances.sql @@ -0,0 +1,9 @@ +CREATE TABLE sqb_location_instances ( + instance_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + connected_at {time} NOT NULL, + last_seen_at {time} NOT NULL, + health_json {json} NOT NULL +); +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index fca5fe59c0..9063a5fedd 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -3,7 +3,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::store::_helpers::migration::{SCHEMA, render, verify_migrations}; +use crate::store::_helpers::migration::{LOCATION_INSTANCES, SCHEMA, render, verify_migrations}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -25,7 +25,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { .await?; let recall_statements = recall_schema(store); let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - let current = verify_migrations(&rows, &digest)?; + let location_digest = sha256_hex(LOCATION_INSTANCES.as_bytes()); + let digests = [digest.clone(), location_digest.clone()]; + let current = verify_migrations(&rows, &digests)?; if rows.is_empty() { for statement in SCHEMA .split(';') @@ -46,5 +48,19 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { VALUES (1, 'control_store', $1, $2, $3)", vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } + if rows.len() < digests.len() { + for statement in LOCATION_INSTANCES + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ + VALUES (2, 'location_instances', $1, $2, $3)", + vec![Parameter::Text(location_digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + } commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index f72b32d6fb..55a6bce444 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -14,6 +14,8 @@ mod store_helpers; mod store_test_types; #[path = "api/test_contract.rs"] mod test_contract; +#[path = "api/test_location_health.rs"] +mod test_location_health; #[path = "api/test_reads.rs"] mod test_reads; #[path = "api/test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs new file mode 100644 index 0000000000..38928875b7 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -0,0 +1,200 @@ +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationHealthObservation, LocationHealthReport, +}; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; +use crate::run_helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::LocationHealthTestCase; + +#[test] +fn given_scoped_location_observations_when_retried_or_read_then_history_and_authority_are_preserved() + { + let test_cases = [ + LocationHealthTestCase { + description: "SQLite health history", + backend: Backend::Sqlite, + expected_observations: 2, + }, + LocationHealthTestCase { + description: "PostgreSQL health history", + backend: Backend::Postgres, + expected_observations: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("location token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/health", api.url); + let other_url = format!("{}/api/locations/{other}/health", api.url); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: "preview #1".into(), + health: LocationHealth::Heartbeat {}, + }; + let before = Utc::now(); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("register") + ); + assert!( + !report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("exact retry") + ); + let changed = LocationHealthReport { + target: "another target".into(), + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&changed) + .send() + .await + .expect("changed retry"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let response = client + .post(&other_url) + .bearer_auth(&token.bearer) + .json(&heartbeat) + .send() + .await + .expect("foreign observation"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .get(&other_url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("foreign history"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .post(&url) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous report"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = client.post(&url).bearer_auth(&token.bearer).json(&json!({ + "event_id": Uuid::now_v7(), "session_id": heartbeat.session_id, "target": heartbeat.target, + "health": { "kind": "heartbeat", "unexpected": "must not disappear" }, + })).send().await.expect("strict health schema"); + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + ..heartbeat + }; + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("refresh existing instance liveness") + ); + let connection = LocationHealthReport { + event_id: Uuid::now_v7(), + health: LocationHealth::Connection { + generation: 1, + healthy: false, + }, + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&connection) + .send() + .await + .expect("unpublished health"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let mut input = snapshot(location_id); + input.target.clone_from(&heartbeat.target); + publish_snapshot(store, &input) + .await + .expect("published generation"); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &connection) + .await + .expect("connection observation") + ); + let response = client + .get(&url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("health history"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let observations: Vec = + response.json().await.expect("typed health history"); + assert_eq!( + observations.len(), + test_case.expected_observations, + "{}: rejected and duplicate reports append nothing", + test_case.description + ); + assert_eq!(observations[0].report, connection); + assert_eq!(observations[1].report, heartbeat); + let events = read_recall_events(store, 0, 100) + .await + .expect("read durable events"); + assert_eq!( + events + .iter() + .filter( + |event| event.get("event_type").and_then(serde_json::Value::as_str) + == Some("location_health") + ) + .count(), + 1, + "heartbeats only update liveness; connection observations remain durable facts" + ); + assert!( + observations + .iter() + .all(|observation| observation.received_at >= before + && observation.received_at <= Utc::now()) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 550618fdb8..59c0610602 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -33,3 +33,9 @@ pub(crate) struct TestApi { pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, pub(crate) task: tokio::task::JoinHandle>, } + +pub(crate) struct LocationHealthTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 9030c6f077..ab2594e4e4 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -241,6 +241,25 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { .await .expect("publish good snapshot"); let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); + connection + .execute_batch( + "DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version=2", + ) + .expect("reconstruct the previous schema without changing existing facts"); + migrate_store(&store) + .await + .expect("upgrade the previous schema"); + migrate_store(&store) + .await + .expect("repeat the migration without changes"); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read upgraded store") + .expect("snapshot survives upgrade") + .record, + good + ); // Fail after immutable bodies were inserted but before switching the current pointer. connection.execute_batch("CREATE TRIGGER fail_snapshot_switch BEFORE UPDATE ON sqb_current_snapshots BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END").expect("install publication failure"); input.generation = 2; @@ -279,7 +298,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { }, SchemaRejectionCase { description: "migration version is unknown", - update_sql: "UPDATE sqb_schema_migrations SET version=2", + update_sql: "UPDATE sqb_schema_migrations SET version=999 WHERE version=2", }, ] { connection diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 8a824ae6e6..0f6a0d3127 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -189,18 +189,8 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny anchored.push(entry); continue; }; - let local_file: bool = match mapping.get_item(ATTACH_TYPE_KEY)? { - None => true, - Some(kind) => kind - .extract::() - .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), - }; let copy: Bound<'_, PyDict> = mapping.copy()?; - if let Some(path_value) = mapping.get_item(ATTACH_PATH_KEY)? - && local_file - && !path_value.extract::().is_ok_and(|text| { - text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) - }) + if let Some(path_value) = local_attach_file(mapping)? && let Some(file) = anchored_file_path(&path_value, project_dir)? { copy.set_item(ATTACH_PATH_KEY, file)?; @@ -210,6 +200,59 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny config.set_item(ATTACH_KEY, pyo3::types::PyList::new(py, anchored)?) } +fn local_attach_file<'py>(mapping: &Bound<'py, PyDict>) -> PyResult>> { + let local = match mapping.get_item(ATTACH_TYPE_KEY)? { + None => true, + Some(kind) => kind + .extract::() + .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), + }; + Ok(mapping.get_item(ATTACH_PATH_KEY)?.filter(|value| { + local + && !value.extract::().is_ok_and(|text| { + text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) + }) + })) +} + +/// Use the resolved adapter connection, including local attachments, not authored TOML guesses. +pub(crate) fn local_database_files( + adapter: &str, + config: &Bound<'_, PyDict>, +) -> PyResult> { + let Some((_, key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter) + else { + return Ok(Vec::new()); + }; + let mut files: Vec = Vec::new(); + if let Some(value) = config.get_item(*key)? + && let Ok(file) = value.extract::() + && !file.is_empty() + && file != DUCKDB_MEMORY_DATABASE + && !file.contains(URL_SCHEME_SEPARATOR) + && !file.starts_with(MOTHERDUCK_PATH_PREFIX) + { + files.push(file); + } + if adapter == DUCKDB_ADAPTER + && let Some(entries) = config.get_item(ATTACH_KEY)? + && let Ok(entries) = entries.try_iter() + { + for entry in entries { + if let Ok(mapping) = entry?.cast_into::() + && let Some(value) = local_attach_file(&mapping)? + && let Ok(file) = value.extract::() + && file != DUCKDB_MEMORY_DATABASE + { + files.push(file); + } + } + } + Ok(files) +} + /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index ae20cf9408..f3a130215b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -28,7 +28,9 @@ use crate::bindings::_helpers::boundary::python_calls::{ }; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::connection::{ + local_database_files, project_connection_config, +}; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, resolution_message, }; @@ -357,6 +359,7 @@ fn build_report( ConnectionCheck::Skipped }; Ok(DebugReport { + local_files: local_database_files(&facts.adapter_name, &connection_config)?, runtime: debug_runtime_lines(&runtime(py)?), configuration: debug_configuration_lines(&facts, &warehouses), providers: debug_provider_lines(&providers(&context.discovered_inputs)?), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index be66d72ee4..eebedd8538 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -39,6 +39,8 @@ class LocationReloadTestCase: target: str expected_generation: int expected_stop_code: int + database: str + expected_connection: bool @dataclass(frozen=True) @@ -46,3 +48,13 @@ class LocationShutdownTestCase: description: str signal: int expected_exit_code: int + minimum_heartbeats: int + blocked_command: str + + +@dataclass(frozen=True) +class LocationDatabaseTestCase: + description: str + database: str + expected_generation: int + connection_toml: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 994e90c8cc..efa304cafd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -79,6 +79,38 @@ def wait_for_log(process: subprocess.Popen[bytes], log: Path, text: str) -> None time.sleep(0.02) +def wait_for_location_health( + api: SnapshotApi, + target: str, + health: dict[str, Any], + process: subprocess.Popen[bytes], + log: Path, + *, + minimum_reports: int = 1, +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + 30 * (minimum_reports - 1) + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + key: tuple[str, str] = (target, json.dumps(health, sort_keys=True)) + observations: dict[tuple[str, str], dict[str, dict[str, Any]]] = {} + while len(observations.get(key, ())) < minimum_reports: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + reports: list[dict[str, Any]] = json.load(response) + for report in reversed(reports): + event_id: str = report["report"]["event_id"] + observed_key: tuple[str, str] = ( + report["report"]["target"], + json.dumps(report["report"]["health"], sort_keys=True), + ) + observations.setdefault(observed_key, {})[event_id] = report + time.sleep(0.02) + return list(observations[key].values())[-1] + + def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: deadline: float = time.monotonic() + 15 while not pidfile.exists(): diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index d2cbbe1f13..26892bc6c0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -9,10 +9,12 @@ from pathlib import Path from typing import Any +import duckdb import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationDatabaseTestCase, LocationReloadTestCase, LocationShutdownTestCase, ) @@ -22,6 +24,7 @@ snapshot_api, wait_for_generation, wait_for_interpreter, + wait_for_location_health, wait_for_log, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -30,8 +33,22 @@ @pytest.mark.parametrize( "test_case", [ - LocationReloadTestCase("ordinary target", "development", 5, int(sys.platform == "win32")), - LocationReloadTestCase("encoded target", "preview #1", 5, int(sys.platform == "win32")), + LocationReloadTestCase( + "healthy connection", + "development", + 5, + int(sys.platform == "win32"), + "warehouse.duckdb", + True, + ), + LocationReloadTestCase( + "failed connection with encoded target", + "preview #1", + 5, + int(sys.platform == "win32"), + "missing/warehouse.duckdb", + False, + ), ], ids=lambda case: case.description, ) @@ -45,7 +62,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) files: dict[str, str] = TYPE_PROOF_FILES | { "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] - + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', "orchestration/orders.py": declaration, } @@ -73,6 +90,17 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ publications: Path = project / "publications.jsonl" with location_process(arguments, log, publications) as process: first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + heartbeat: dict[str, Any] = wait_for_location_health( + api, test_case.target, {"kind": "heartbeat"}, process, log + ) + initial_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + assert initial_health["report"]["session_id"] == heartbeat["report"]["session_id"] staging: Path = project / "models/stg_orders.sql" for value in (11, 12, 13): _ = staging.write_text( @@ -87,6 +115,9 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 3}, process, log + ) assert current_snapshot(api, test_case.target)["record"] == second["record"] _ = (project / "orchestration/orders.py").write_text( declaration.replace("Build orders.", "Refresh order totals.") @@ -121,6 +152,17 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ api, test_case.target, test_case.expected_generation, process, log ) assert restarted["nodes"] == third["nodes"] + restarted_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 5, "healthy": test_case.expected_connection}, + process, + log, + ) + assert ( + restarted_health["report"]["session_id"] != initial_health["report"]["session_id"] + ) + assert restarted_health["received_at"] >= initial_health["received_at"] # Redirected output and compiler cache writes must not feed the input watcher. time.sleep(1) assert current_snapshot(api, test_case.target)["record"] == restarted["record"] @@ -132,19 +174,28 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ assert [record["generation"] for record in records] == [1, 2, 3, 4, 5] assert api.bearer not in log.read_text() + publications.read_text() assert not (project / ".sqlbuild/control.sqlite3").exists() - assert not (project / "warehouse.duckdb").exists() + assert (project / "warehouse.duckdb").exists() == test_case.expected_connection @pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") @pytest.mark.parametrize( "test_case", [ - LocationShutdownTestCase("terminate compiling interpreter", signal.SIGTERM, 0), - LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0), + LocationShutdownTestCase( + "terminate compiling interpreter after repeated heartbeats", + signal.SIGTERM, + 0, + 2, + "publish", + ), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "publish"), + LocationShutdownTestCase( + "terminate connection checking interpreter", signal.SIGTERM, 0, 1, "debug" + ), ], ids=lambda case: case.description, ) -def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( +def test_given_a_busy_location_when_stopped_then_its_interpreter_is_reaped( test_case: LocationShutdownTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project: Path = tmp_path / "project" @@ -156,10 +207,11 @@ def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', - "orchestration/slow.py": ( - "import os, time\nfrom pathlib import Path\n" - f"Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" - "time.sleep(60)\n" + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if {test_case.blocked_command!r} in sys.argv:\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + " time.sleep(60)\n" ), }, ) @@ -181,7 +233,87 @@ def test_given_a_compiling_location_when_stopped_then_its_interpreter_is_reaped( ] with location_process(arguments, log, tmp_path / "publications.jsonl") as process: interpreter: int = wait_for_interpreter(process, child_pid, log) + _ = wait_for_location_health( + api, + "development", + {"kind": "heartbeat"}, + process, + log, + minimum_reports=test_case.minimum_heartbeats, + ) process.send_signal(test_case.signal) assert process.wait(timeout=5) == test_case.expected_exit_code with pytest.raises(ProcessLookupError): os.kill(interpreter, 0) + + +@pytest.mark.parametrize( + "test_case", + [ + LocationDatabaseTestCase( + "custom database extension", "warehouse.db", 1, 'database = "warehouse.db"' + ), + LocationDatabaseTestCase( + "extensionless database", "warehouse", 1, 'database = "warehouse"' + ), + LocationDatabaseTestCase( + "custom attached database", + "inventory.db", + 1, + 'database = ":memory:"\nattach = [{path = "inventory.db", alias = "inventory"}]', + ), + ], + ids=lambda case: case.description, +) +def test_given_a_location_using_a_local_database_when_it_is_created_and_written_then_only_source_edits_reload( + test_case: LocationDatabaseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\n{test_case.connection_toml}\n" + + '\n[targets.development]\nschema = "main"\n', + }, + ) + database: Path = project / test_case.database + assert not database.exists() + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + baseline: dict[str, Any] = wait_for_generation( + api, "development", test_case.expected_generation, process, log + ) + _ = wait_for_location_health( + api, + "development", + {"kind": "connection", "generation": 1, "healthy": True}, + process, + log, + ) + assert database.is_file() + with duckdb.connect(str(database)) as connection: + _ = connection.execute( + "CREATE TABLE orders (order_id INTEGER); INSERT INTO orders VALUES (17), (23)" + ) + assert connection.execute("SELECT sum(order_id) FROM orders").fetchone() == (40,) + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] From 13b990d5a59e574456d9b0b14579a679b35cbe56 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:51:54 +0000 Subject: [PATCH 032/138] feat: keep code location compilers warm --- .../sqlbuild-cli/src/argv/command_table.json | 4 +- .../src/native_commands/_helpers/execution.rs | 94 ++++++++++++++++--- .../main/_location_compiler.rs | 83 ++++++++++++++++ .../src/native_commands/main/_run_location.rs | 37 ++++---- .../src/native_commands/main/mod.rs | 1 + .../src/native_commands/models.rs | 19 ++++ .../src/bindings/_helpers/cli/entry.rs | 29 +++++- .../bindings/_helpers/cli/observability.rs | 17 +++- .../_helpers/debug_command/run_debug.rs | 27 ++++-- .../_helpers/location_command/command.rs | 94 ++++++++++++------- .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/runtime.rs | 50 ++++++++++ .../src/bindings/_helpers/native_host/host.rs | 7 ++ .../publication/test_reload.py | 58 +++++++++--- 14 files changed, 426 insertions(+), 95 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f63070d273..1dd2277e11 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -106,8 +106,8 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish", "serve"], - "help": "publish one snapshot or watch and reload the project", + "choices": ["publish", "serve", "runtime"], + "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", "group": 0 }, { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 30241401e9..a9d41344e3 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -10,13 +10,14 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use tokio::process::{Child, Command}; -use tokio::sync::watch; +use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinSet; use tokio::time::MissedTickBehavior; use uuid::Uuid; @@ -44,10 +45,82 @@ use crate::native_commands::constants::{ use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationServe, LostStdout, NativeCommand, NativeInvocation, - ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, + LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +/// Request one publication without transferring interpreter ownership to a cancellable future. +pub(crate) async fn compile_generation( + requests: &mpsc::Sender, + generation: i64, + sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], + stopped: impl Future>, +) -> Result, ControlError> { + let (reply, publication) = oneshot::channel(); + requests + .send(LocationCompileRequest { + generation, + python_sources: sources + .iter() + .filter(|stamp| stamp.relative_path.ends_with(".py")) + .cloned() + .collect(), + reply, + }) + .await + .map_err(|_| ControlError::Invalid("location compiler stopped".into()))?; + tokio::pin!(stopped); + let result = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = publication => result.map_err(|_| ControlError::Invalid("location compiler stopped before replying".into()))?, + }; + let record = match result { + Ok(record) => record, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location compiler failed; generation {generation} may already be published: {error}\n" + ), + )])); + None + } + }; + let published = record.is_some(); + if let Some(record) = record { + let output = serde_json::to_string(&record) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { + return Err(ControlError::Invalid(format!( + "Snapshot {} was published, but result delivery failed", + record.snapshot_id + ))); + } + } + Ok(Some(published)) +} + +/// The child is retained by its owning task while this exchange can be cancelled. +pub(crate) async fn exchange_location_generation( + compiler: &mut LocationCompiler, + generation: i64, +) -> Result, ControlError> { + compiler + .stdin + .write_all(format!("{generation}\n").as_bytes()) + .await?; + compiler.stdin.flush().await?; + let mut line = String::new(); + if compiler.stdout.read_line(&mut line).await? == 0 { + return Err(ControlError::Invalid( + "location interpreter exited before replying".into(), + )); + } + serde_json::from_str(&line).map_err(|_| { + ControlError::Invalid("location interpreter returned an invalid publication record".into()) + }) +} + /// Health IO remains native even while a user-code child holds its interpreter's GIL. pub(crate) async fn report_location_health( options: &LocationServe, @@ -150,27 +223,22 @@ pub(crate) async fn location_generation( } /// The installed interpreter owns user imports; the supervisor never imports Python. -pub(crate) fn publish_child( - options: &LocationServe, - script: &Path, - generation: i64, -) -> io::Result { +pub(crate) fn location_runtime_child(options: &LocationServe, script: &Path) -> io::Result { let mut command = location_child(options, script); command .args([ "location", - "publish", + "runtime", "--server", &options.server_url, "--location-id", ]) .arg(options.location_id.to_string()) - .args(["--target", &options.target, "--generation"]) - .arg(generation.to_string()); + .args(["--target", &options.target]); if let Some(image) = &options.image_id { command.args(["--image-id", image]); } - command.stdout(Stdio::inherit()).spawn() + command.stdin(Stdio::piped()).stdout(Stdio::piped()).spawn() } /// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs new file mode 100644 index 0000000000..2a20ef09b2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs @@ -0,0 +1,83 @@ +//! One task owns a warm interpreter; cancelled requests and source restarts always reap it. + +use std::path::Path; + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::BufReader; +use tokio::sync::mpsc; + +use crate::native_commands::_helpers::execution::{ + exchange_location_generation, location_runtime_child, +}; +use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe}; + +pub(crate) async fn run_location_compiler( + options: &LocationServe, + script: &Path, + mut requests: mpsc::Receiver, +) -> Result<(), ControlError> { + let mut compiler: Option = None; + let result = async { + while let Some(mut request) = requests.recv().await { + if compiler + .as_ref() + .is_some_and(|compiler| compiler.python_sources != request.python_sources) + { + stop_compiler(compiler.take()).await?; + } + if compiler.is_none() { + compiler = Some(open_compiler(options, script, request.python_sources)?); + } + let active = compiler + .as_mut() + .ok_or_else(|| ControlError::Invalid("location compiler is unavailable".into()))?; + let result = tokio::select! { + _ = request.reply.closed() => break, + result = exchange_location_generation(active, request.generation) => result, + }; + let failed = result.is_err(); + let _ = request + .reply + .send(result.map_err(|error| error.to_string())); + if failed { + stop_compiler(compiler.take()).await?; + } + } + Ok(()) + } + .await; + stop_compiler(compiler).await?; + result +} + +fn open_compiler( + options: &LocationServe, + script: &Path, + python_sources: Vec, +) -> Result { + let mut child = location_runtime_child(options, script)?; + let stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?; + let stdout = child.stdout.take().ok_or_else(|| { + ControlError::Invalid("location interpreter output is unavailable".into()) + })?; + Ok(LocationCompiler { + child, + stdin, + stdout: BufReader::new(stdout), + python_sources, + }) +} + +async fn stop_compiler(compiler: Option) -> Result<(), ControlError> { + if let Some(mut compiler) = compiler { + if compiler.child.try_wait()?.is_none() { + compiler.child.kill().await?; + } else { + compiler.child.wait().await?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index b7bff31ea3..0c5060dab2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -6,19 +6,21 @@ use std::time::Duration; use sqlbuild_orchestration::platform::models::LocationHealth; use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::mpsc; use tokio::time::{Instant, MissedTickBehavior}; use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - check_location_connection, location_generation, location_heartbeats, publish_child, + check_location_connection, compile_generation, location_generation, location_heartbeats, report_location_health, shutdown_signal, }; use crate::native_commands::_helpers::paths::{ exclude_location_databases, location_watch, observe_location_inputs, }; use crate::native_commands::_helpers::terminal::write_output; -use crate::native_commands::models::LocationServe; +use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::models::{LocationCompileRequest, LocationServe}; pub(super) async fn run_location( options: LocationServe, @@ -26,17 +28,23 @@ pub(super) async fn run_location( bearer: &str, ) -> Result<(), ControlError> { let session_id = Uuid::now_v7(); - tokio::select! { - result = watch_location(&options, script, bearer, session_id) => result, - result = location_heartbeats(&options, bearer, session_id) => result, - } + let (requests, receiver) = mpsc::channel(1); + let serving = async { + tokio::select! { + result = watch_location(&options, script, (bearer, session_id), requests) => result, + result = location_heartbeats(&options, bearer, session_id) => result, + } + }; + let (serving, compiler) = + tokio::join!(serving, run_location_compiler(&options, script, receiver)); + serving.and(compiler) } async fn watch_location( options: &LocationServe, script: &Path, - bearer: &str, - session_id: Uuid, + (bearer, session_id): (&str, Uuid), + requests: mpsc::Sender, ) -> Result<(), ControlError> { let mut watch = location_watch(&options.project_dir); let mut clock = tokio::time::interval(Duration::from_millis(250)); @@ -72,15 +80,12 @@ async fn watch_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let mut child = publish_child(options, script, generation)?; - let result = tokio::select! { - result = &mut stopped => { - child.kill().await?; - return result; - }, - result = child.wait() => result?, + let Some(published) = + compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? + else { + return Ok(()); }; - if result.success() { + if published { watch.retry_at = None; let Some(files) = check_published_location( options, diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index ac5f9232e5..66a0b76d88 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,4 +1,5 @@ pub(super) mod _cost_report_options; +pub(super) mod _location_compiler; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f3ae56696a..152ede9b5e 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -148,6 +148,25 @@ pub(crate) struct LocationWatch { pub(crate) retry_at: Option, } +/// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. +#[derive(Debug)] +pub(crate) struct LocationCompileRequest { + pub(crate) generation: i64, + pub(crate) python_sources: Vec, + pub(crate) reply: tokio::sync::oneshot::Sender< + Result, String>, + >, +} + +/// Only this task owns the isolated interpreter and its private stdio protocol. +#[derive(Debug)] +pub(crate) struct LocationCompiler { + pub(crate) child: tokio::process::Child, + pub(crate) stdin: tokio::process::ChildStdin, + pub(crate) stdout: tokio::io::BufReader, + pub(crate) python_sources: Vec, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 3962b79206..8abf67a87f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -122,9 +122,7 @@ fn print_object(value: &Bound<'_, PyAny>, stream: Option<&str>, end: Option<&str if let Some(end) = end { keywords.set_item("end", end)?; } - py.import("builtins")? - .getattr("print")? - .call((value,), Some(&keywords))?; + imported(py, "builtins", "print")?.call((value,), Some(&keywords))?; Ok(()) } @@ -766,6 +764,31 @@ fn parsed_invocation<'py>( Ok(exit_code) }; } + let namespace = args.getattr("__dict__")?.cast_into::()?; + let command: Option = namespace + .get_item("command")? + .map(|value| value.extract()) + .transpose()?; + let location_command: Option = namespace + .get_item("location_command")? + .map(|value| value.extract()) + .transpose()?; + if command.as_deref() == Some("location") && location_command.as_deref() == Some("runtime") { + return within(&host::redirect_stdout_to_stderr(py)?, |_| { + execute_parsed_invocation(args, handlers, identity, use_color) + }); + } + execute_parsed_invocation(args, handlers, identity, use_color) +} + +/// Runtime stream routing must cover extension discovery, invocation scopes and their teardown. +fn execute_parsed_invocation<'py>( + args: Bound<'py, PyAny>, + handlers: Option>, + identity: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py = args.py(); let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); let recall: InvocationRecall = if policy.recalled { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 6a26b63e16..0f48c13014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,7 +15,7 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::progress::NativeProgressProjector; use crate::bindings::_helpers::build_progress::progress::machine_output_mode; use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; @@ -337,10 +337,17 @@ impl Observed { ) -> PyResult<()> { let mut command_output_sinks: bool = false; if native_field(args, "command")?.ne(LINEAGE_COMMAND)? { - let extensions: Bound<'_, PyAny> = call_with( - &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, - &[("project_dir", project_dir.clone())], - )?; + let discover = || { + call_with( + &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, + &[("project_dir", project_dir.clone())], + ) + }; + let extensions: Bound<'_, PyAny> = if machine_output_mode(args)? { + within(&host::redirect_stdout_to_stderr(py)?, |_| discover())? + } else { + discover()? + }; let lifecycle_replacement: Bound<'_, PyAny> = native_field(&extensions, "recall_lifecycle_sink")?; let output_replacement: Bound<'_, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index f3a130215b..c0e0910921 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -384,15 +385,23 @@ pub(crate) fn run_debug_command( } else { project_dir.extract()? }; - let report: DebugReport = build_report( - py, - &project_dir, - ( - arguments.value("selected_target")?, - !arguments.flag("no_connection")?, - ), - )?; - let text: String = if arguments.flag("json_output")? { + let json_output = arguments.flag("json_output")?; + let build = || { + build_report( + py, + &project_dir, + ( + arguments.value("selected_target")?, + !arguments.flag("no_connection")?, + ), + ) + }; + let report: DebugReport = if json_output { + within(&redirect_stdout_to_stderr(py)?, |_| build())? + } else { + build()? + }; + let text: String = if json_output { format_debug_json(&report) + "\n" } else { let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 1ec1b8a2ee..e5673f87ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Py use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; @@ -17,6 +17,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; +use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -130,14 +131,11 @@ fn compile_location<'py>( Ok(snapshot) } -/// One publication is also the isolated compiler boundary a reloading location can invoke. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn run_location_command<'py>( +pub(super) fn location_arguments<'py>( py: Python<'py>, options: Option>, -) -> PyResult { - let arguments = KeywordArguments::read( +) -> PyResult> { + KeywordArguments::read( py, "run_location_command", options, @@ -151,34 +149,37 @@ pub(crate) fn run_location_command<'py>( ], &["image_id", "action"], ), - )?; - let publication = (|| { - if arguments - .value("action")? - .extract::>()? - .is_some_and(|action| action != PUBLISH_ACTION) - { - return Err(PyValueError::new_err( - "location serve must run through the native sqb executable", - )); - } - let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { - PyValueError::new_err( - "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", - ) - })?; - if bearer.trim().is_empty() { - return Err(PyValueError::new_err( - "SQLBUILD_LOCATION_TOKEN must not be blank", - )); - } - let directory = arguments - .value("project_dir")? - .extract::()?; - let input = metadata(py, &arguments, &directory)?; - let snapshot = compile_location(py, &arguments, &input)?; - status(py, "Publishing location snapshot...")?; - let record = snapshot.publication( + ) +} + +pub(super) fn publish_location( + py: Python<'_>, + arguments: &KeywordArguments<'_>, +) -> PyResult { + if arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action != PUBLISH_ACTION) + { + return Err(PyValueError::new_err( + "location serve must run through the native sqb executable", + )); + } + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + PyValueError::new_err("SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential") + })?; + if bearer.trim().is_empty() { + return Err(PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must not be blank", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, arguments, &directory)?; + let snapshot = compile_location(py, arguments, &input)?; + status(py, "Publishing location snapshot...")?; + let record = snapshot.publication( py, &arguments.value("server_url")?.extract::()?, &bearer, @@ -190,6 +191,28 @@ pub(crate) fn run_location_command<'py>( }; PyValueError::new_err(message) })?; + status(py, "Location snapshot published.")?; + Ok(record) +} + +/// One-shot publication and the supervisor's private stdio runtime share the same compiler path. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_location_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments = location_arguments(py, options.clone())?; + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("runtime") + { + return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); + } + let publication = (|| { + let record = publish_location(py, &arguments)?; let output = serde_json::to_string(&record) .map_err(|error| PyValueError::new_err(error.to_string()))?; write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { @@ -198,7 +221,6 @@ pub(crate) fn run_location_command<'py>( record.snapshot_id )) })?; - status(py, "Location snapshot published.")?; Ok(0) })(); if publication.is_err() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 8aa0ebb43e..f80ce86450 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod command; mod constants; +mod runtime; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs new file mode 100644 index 0000000000..6a0505dbda --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -0,0 +1,50 @@ +//! A location keeps its interpreter warm; only the supervisor writes generation requests. + +use std::io::{self, Write}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::bindings::_helpers::boundary::python_calls::within; +use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; + +pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { + options.set_item("action", "publish")?; + let redirect = redirect_stdout_to_stderr(py)?; + within(&redirect, |_| { + loop { + let line = py.detach(|| -> io::Result> { + let mut line = String::new(); + Ok((io::stdin().read_line(&mut line)? != 0).then_some(line)) + })?; + let Some(line) = line else { + break; + }; + let generation: i64 = serde_json::from_str(&line) + .map_err(|_| PyValueError::new_err("runtime expects a JSON generation per line"))?; + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_console( + py, + "stderr", + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + let output = serde_json::to_string(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| -> io::Result<()> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{output}")?; + stdout.flush() + })?; + } + Ok(0) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index 796bb304d7..babcc38c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -73,6 +73,13 @@ pub(crate) fn write_console(py: Python<'_>, stream: &str, text: &str) -> PyResul Ok(()) } +/// Keep user prints off machine-readable stdout while running project code. +pub(crate) fn redirect_stdout_to_stderr(py: Python<'_>) -> PyResult> { + py.import("contextlib")? + .getattr("redirect_stdout")? + .call1((py.import("sys")?.getattr("stderr")?,)) +} + /// `log_debug_event(logger=logging.getLogger(logger), message=message)`. pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { let kwargs: Bound<'_, PyDict> = PyDict::new(py); diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index 26892bc6c0..abc6cdea92 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -36,7 +36,7 @@ LocationReloadTestCase( "healthy connection", "development", - 5, + 7, int(sys.platform == "win32"), "warehouse.duckdb", True, @@ -44,7 +44,7 @@ LocationReloadTestCase( "failed connection with encoded target", "preview #1", - 5, + 7, int(sys.platform == "win32"), "missing/warehouse.duckdb", False, @@ -56,8 +56,12 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project: Path = tmp_path / "project" + compiler_pid: Path = tmp_path / "compiler.pid" declaration: str = ( "from sqlbuild.orchestration import job\n" + "import os\nfrom pathlib import Path\n" + f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" + 'print("Orders declaration loaded.")\n' 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' ) files: dict[str, str] = TYPE_PROOF_FILES | { @@ -65,6 +69,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', "orchestration/orders.py": declaration, + "sinks/orders.py": 'print("Loading orders sink", flush=True)\n', } write_files(project_dir=project, files=files) _ = subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) @@ -90,6 +95,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ publications: Path = project / "publications.jsonl" with location_process(arguments, log, publications) as process: first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + first_interpreter: int = int(compiler_pid.read_text()) heartbeat: dict[str, Any] = wait_for_location_health( api, test_case.target, {"kind": "heartbeat"}, process, log ) @@ -110,19 +116,34 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) second: dict[str, Any] = wait_for_generation(api, test_case.target, 2, process, log) assert second["record"]["snapshot_id"] != first["record"]["snapshot_id"] + assert int(compiler_pid.read_text()) == first_interpreter assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} - _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") - wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = staging.write_text("SELECT FROM invalid SQL !\n") _ = wait_for_location_health( api, test_case.target, {"kind": "compile_failed", "generation": 3}, process, log ) assert current_snapshot(api, test_case.target)["record"] == second["record"] + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(14 AS INTEGER)") + ) + recovered: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + assert int(compiler_pid.read_text()) == first_interpreter + assert "CAST(14 AS INTEGER)" in recovered["nodes"]["model:stg_orders"]["raw_sql"] + + _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 4}, process, log + ) + assert current_snapshot(api, test_case.target)["record"] == recovered["record"] _ = (project / "orchestration/orders.py").write_text( declaration.replace("Build orders.", "Refresh order totals.") ) - third: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + third: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + replacement_interpreter: int = int(compiler_pid.read_text()) + assert replacement_interpreter != first_interpreter assert ( third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." ) @@ -143,19 +164,34 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ check=True, capture_output=True, ) - committed: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + committed: dict[str, Any] = wait_for_generation(api, test_case.target, 5, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter assert committed["nodes"] == third["nodes"] + _ = (project / "sqlbuild_project.toml").write_text( + files["sqlbuild_project.toml"].replace('schema = "main"', 'schema = "reporting"') + ) + configured: dict[str, Any] = wait_for_generation(api, test_case.target, 6, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter + assert ( + configured["nodes"]["model:orders"]["definition"]["destination"]["schema"] + == "reporting" + ) + assert ( + configured["nodes"]["model:orders"]["raw_sql"] + == committed["nodes"]["model:orders"]["raw_sql"] + ) process.terminate() assert process.wait(timeout=5) == test_case.expected_stop_code with location_process(arguments, log, publications) as process: restarted: dict[str, Any] = wait_for_generation( api, test_case.target, test_case.expected_generation, process, log ) - assert restarted["nodes"] == third["nodes"] + assert int(compiler_pid.read_text()) != replacement_interpreter + assert restarted["nodes"] == configured["nodes"] restarted_health: dict[str, Any] = wait_for_location_health( api, test_case.target, - {"kind": "connection", "generation": 5, "healthy": test_case.expected_connection}, + {"kind": "connection", "generation": 7, "healthy": test_case.expected_connection}, process, log, ) @@ -171,7 +207,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ records: list[dict[str, Any]] = [ json.loads(line) for line in publications.read_text().splitlines() ] - assert [record["generation"] for record in records] == [1, 2, 3, 4, 5] + assert [record["generation"] for record in records] == [1, 2, 3, 4, 5, 6, 7] assert api.bearer not in log.read_text() + publications.read_text() assert not (project / ".sqlbuild/control.sqlite3").exists() assert (project / "warehouse.duckdb").exists() == test_case.expected_connection @@ -186,9 +222,9 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ signal.SIGTERM, 0, 2, - "publish", + "runtime", ), - LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "publish"), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "runtime"), LocationShutdownTestCase( "terminate connection checking interpreter", signal.SIGTERM, 0, 1, "debug" ), From 48b08b599e9cad2444a73615b2f8806c7336aec3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:15:02 +0000 Subject: [PATCH 033/138] feat: process scoped outbound location work --- .../sqlbuild-cli/src/argv/command_table.json | 96 ++++++ .../src/native_commands/_helpers/decision.rs | 21 ++ .../src/native_commands/_helpers/paths.rs | 2 +- .../native_commands/main/_location_work.rs | 135 ++++++++ .../src/native_commands/main/_run_admin.rs | 9 + .../src/native_commands/main/_run_location.rs | 41 ++- .../src/native_commands/main/mod.rs | 1 + .../src/native_commands/models.rs | 3 + .../tests/native_commands/test_control.rs | 24 ++ .../src/api/_helpers/location_work.rs | 65 ++++ .../src/api/_helpers/mod.rs | 1 + .../src/api/main/serve_api.rs | 12 +- .../sqlbuild-orchestration/src/api/models.rs | 7 +- .../src/platform/_helpers/location_work.rs | 40 +++ .../src/platform/_helpers/mod.rs | 2 + .../src/platform/_helpers/work_client.rs | 51 +++ .../src/platform/main/claim_location_work.rs | 63 ++++ .../platform/main/complete_location_work.rs | 57 ++++ .../platform/main/connect_location_work.rs | 21 ++ .../src/platform/main/location_work.rs | 20 ++ .../src/platform/main/mod.rs | 5 + .../platform/main/request_location_work.rs | 81 +++++ .../src/platform/models.rs | 79 +++++ .../src/store/_helpers/migration.rs | 1 + .../src/store/location_work.sql | 20 ++ .../src/store/main/migrate_store.rs | 24 +- crates/sqlbuild-orchestration/tests/api.rs | 4 + .../tests/api/test_location_work.rs | 302 ++++++++++++++++++ .../tests/api/test_types.rs | 6 + .../tests/api/work/helpers.rs | 20 ++ .../tests/snapshots/helpers.rs | 3 +- .../publication/_test_types.py | 7 + .../compiled_project/publication/helpers.py | 19 ++ .../compiled_project/publication/test_work.py | 158 +++++++++ 34 files changed, 1384 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/store/location_work.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_location_work.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/work/helpers.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 1dd2277e11..d86b9c0ac6 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -523,6 +523,92 @@ ] } }, + { + "name": "check-location", + "parser": { + "prog": "sqb admin check-location", + "description": "Request an outbound connection check on a published location using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": ["--request-id"], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "location-work", + "parser": { + "prog": "sqb admin location-work", + "description": "Read an outbound location request and its immutable result using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "REQUEST_ID", + "required": true, + "help": "location work request UUID", + "group": 0 + } + ] + } + }, { "name": "cancel-run", "parser": { @@ -705,6 +791,16 @@ "metavar": "cancel-run", "help": "request run cancellation with an atomic audit and retry identity" }, + { + "dest": "check-location", + "metavar": "check-location", + "help": "request an outbound location connection check with an atomic audit" + }, + { + "dest": "location-work", + "metavar": "location-work", + "help": "read a location work request and its result" + }, { "dest": "change-trigger", "metavar": "change-trigger", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 9e79832001..aae4693e44 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, Utc}; use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; use sqlbuild_orchestration::runs::models::JobRunRequest; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -68,6 +69,26 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result Ok(ControlAdmin::LocationWork(id)), + Some("check-location") => { + let request_id = + Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("--request-id must be a UUID".into()))?; + let target = text(namespace, "admin_target").unwrap_or_default(); + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if target.trim().is_empty() || reason.trim().is_empty() { + return Err(ControlError::Invalid( + "target and reason must not be blank".into(), + )); + } + Ok(ControlAdmin::RequestLocationWork(LocationWorkRequest { + request_id, + location_id: id, + target, + reason, + work: LocationWork::CheckConnection {}, + })) + } Some("change-trigger" | "cancel-run" | "request-run") => { let identity_option = if action.as_deref() == Some("request-run") { "--run-id" diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index e51e498caa..78b96ec94b 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -70,7 +70,7 @@ fn location_rules() -> SnapshotRules { } /// Git commits matter to publication metadata even when no working-tree file changes. -fn location_inputs( +pub(crate) fn location_inputs( root: &Path, rules: &SnapshotRules, local_files: &[PathBuf], diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs new file mode 100644 index 0000000000..281b8bb686 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -0,0 +1,135 @@ +//! Named outbound work shares the watcher's generation boundary, never its interpreter. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::Utc; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::models::{ + LocationWork, LocationWorkClient, LocationWorkOutcome, LocationWorkPoll, LocationWorkResult, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::time::Instant; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::check_location_connection; +use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::models::{LocationServe, LocationWatch}; + +/// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. +pub(super) async fn run_location_work( + (options, script): (&LocationServe, &Path), + (client, poll, watch): (&LocationWorkClient, &LocationWorkPoll, &LocationWatch), + generation: Option, + stopped: impl Future>, +) -> Result>, ControlError> { + tokio::pin!(stopped); + let claim = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = client.poll(poll) => match result { + Ok(claim) => claim, + Err(error) => { + work_error(&error)?; + return Ok(Some(Vec::new())); + }, + }, + }; + let Some(claim) = claim else { + return Ok(Some(Vec::new())); + }; + let deadline = Instant::now() + + (claim.lease_expires_at - Utc::now()) + .to_std() + .unwrap_or_default(); + let mut files: Vec = Vec::new(); + let outcome = if generation == Some(claim.generation) { + drop(write_output(&[( + Stream::Stderr, + "Checking requested location connection...\n", + )])); + let mut shutting_down = false; + let expires = async { + tokio::select! { + result = &mut stopped => { shutting_down = true; result }, + () = tokio::time::sleep_until(deadline) => Ok(()), + } + }; + let checked = match claim.request.work { + LocationWork::CheckConnection {} => { + check_location_connection(options, script, expires).await? + } + }; + let Some((healthy, local_files)) = checked else { + drop(write_output(&[( + Stream::Stderr, + "Location work stopped before completion; no result reported.\n", + )])); + return Ok((!shutting_down).then_some(files)); + }; + files = local_files; + let mut expected = watch.observed.clone(); + expected + .0 + .retain(|stamp| !files.contains(&options.project_dir.join(&stamp.relative_path))); + let mut excluded = watch.local_files.clone(); + excluded.extend(files.clone()); + if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { + LocationWorkOutcome::Connection { healthy } + } else { + LocationWorkOutcome::Superseded {} + } + } else { + LocationWorkOutcome::Superseded {} + }; + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome, + }; + while Instant::now() < deadline { + let completed = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep_until(deadline) => break, + result = client.complete(claim.request.request_id, &result) => result, + }; + match completed { + Ok(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location work completed.\n", + )])); + return Ok(Some(files)); + } + Err(error) => { + work_error(&error)?; + if matches!(error, ClientError::Rejected(status) if status.is_client_error()) { + break; + } + } + } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep(Duration::from_secs(1)) => {}, + } + } + drop(write_output(&[( + Stream::Stderr, + "Location work acknowledgement expired; check the server result before retrying.\n", + )])); + Ok(Some(files)) +} + +fn work_error(error: &ClientError) -> Result<(), ControlError> { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!("Location work request failed: {error}\n"), + )])); + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 9c2bbde856..7cfcd0074f 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -8,6 +8,8 @@ use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; +use sqlbuild_orchestration::platform::main::location_work::location_work; +use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; @@ -49,6 +51,13 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = request_location_work(&store, &context, &request).await?; + Ok(json!({"request_id": request.request_id, "applied": applied})) + } + ControlAdmin::LocationWork(request_id) => { + Ok(json!(location_work(&store, request_id).await?)) + } ControlAdmin::CancelRun { event_id, run_id, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 0c5060dab2..7fc5eb449e 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -4,7 +4,10 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::time::Duration; -use sqlbuild_orchestration::platform::models::LocationHealth; +use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationWorkClient, LocationWorkPoll, +}; use sqlbuild_orchestration::store::errors::ControlError; use tokio::sync::mpsc; use tokio::time::{Instant, MissedTickBehavior}; @@ -20,6 +23,7 @@ use crate::native_commands::_helpers::paths::{ }; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::main::_location_work::run_location_work; use crate::native_commands::models::{LocationCompileRequest, LocationServe}; pub(super) async fn run_location( @@ -28,10 +32,16 @@ pub(super) async fn run_location( bearer: &str, ) -> Result<(), ControlError> { let session_id = Uuid::now_v7(); + let client = connect_location_work((&options.server_url, bearer), options.location_id) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let poll = LocationWorkPoll { + session_id, + target: options.target.clone(), + }; let (requests, receiver) = mpsc::channel(1); let serving = async { tokio::select! { - result = watch_location(&options, script, (bearer, session_id), requests) => result, + result = watch_location(&options, script, (bearer, &client, &poll), requests) => result, result = location_heartbeats(&options, bearer, session_id) => result, } }; @@ -43,10 +53,12 @@ pub(super) async fn run_location( async fn watch_location( options: &LocationServe, script: &Path, - (bearer, session_id): (&str, Uuid), + (bearer, client, poll): (&str, &LocationWorkClient, &LocationWorkPoll), requests: mpsc::Sender, ) -> Result<(), ControlError> { let mut watch = location_watch(&options.project_dir); + let mut work_at = Instant::now() + Duration::from_secs(2); + let mut published_generation = None; let mut clock = tokio::time::interval(Duration::from_millis(250)); clock.set_missed_tick_behavior(MissedTickBehavior::Skip); let stopped = shutdown_signal(); @@ -63,8 +75,26 @@ async fn watch_location( let ready; (watch, ready) = observe_location_inputs(&options.project_dir, watch); if !ready { + if Instant::now() >= work_at + && published_generation.is_some() + && watch.attempted.as_ref() == Some(&watch.observed) + { + let Some(files) = run_location_work( + (options, script), + (client, poll, &watch), + published_generation, + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + work_at = Instant::now() + Duration::from_secs(2); + } continue; } + published_generation = None; let generation = tokio::select! { result = &mut stopped => return result, generation = location_generation(options, bearer) => generation?, @@ -87,10 +117,11 @@ async fn watch_location( }; if published { watch.retry_at = None; + published_generation = Some(generation); let Some(files) = check_published_location( options, script, - (bearer, session_id, generation), + (bearer, poll.session_id, generation), stopped.as_mut(), ) .await? @@ -102,7 +133,7 @@ async fn watch_location( watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); tokio::select! { result = &mut stopped => return result, - () = report_location_health(options, bearer, session_id, LocationHealth::CompileFailed { generation }) => {}, + () = report_location_health(options, bearer, poll.session_id, LocationHealth::CompileFailed { generation }) => {}, } drop(write_output(&[( Stream::Stderr, diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 66a0b76d88..a1333f51a6 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,5 +1,6 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; +pub(super) mod _location_work; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 152ede9b5e..619784b8bd 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -6,6 +6,7 @@ use chrono::{DateTime, Utc}; use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::JobRunRequest; use uuid::Uuid; @@ -18,6 +19,8 @@ pub(crate) enum ControlAdmin { IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), RequestJobRun(JobRunRequest), + RequestLocationWork(LocationWorkRequest), + LocationWork(Uuid), CancelRun { event_id: Uuid, run_id: Uuid, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 2347a94d9a..215540fdb2 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -49,6 +49,30 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin request-run ", expected_stderr: "", }, + CommandTestCase { + description: "native location work help", + arguments: &["admin", "check-location", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin check-location ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank location work reason rejected before store initialization", + arguments: &[ + "admin", + "check-location", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + "--request-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and reason must not be blank\n", + }, CommandTestCase { description: "blank job rejected before store initialization", arguments: &[ diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs new file mode 100644 index 0000000000..6e48859e78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs @@ -0,0 +1,65 @@ +//! Locations may poll and complete only their own work; enqueueing stays a trusted-owner action. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::claim_location_work::claim_location_work; +use crate::platform::main::complete_location_work::complete_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkPoll, LocationWorkRecord, LocationWorkResult, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/poll", + params(("location_id" = Uuid, Path)), request_body = LocationWorkPoll, + responses((status = 200, body = Option), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn poll( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(poll): Json, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(claim_location_work(&store, location_id, &poll).await?)) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), request_body = LocationWorkResult, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Work claim expired or was replaced")), security(("bearer" = [])))] +pub(crate) async fn complete( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, + Json(result): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: complete_location_work(&store, (location_id, request_id), &result).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn result( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_work(&store, request_id).await?.filter( + |record| record.request_json.location_id == location_id, + ))) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 4b3f4f829e..996c7546bb 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod authentication; pub(crate) mod client; pub(crate) mod location_health; +pub(crate) mod location_work; pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 0a7ba5c5c2..f2ce27445f 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -5,7 +5,9 @@ use axum::routing::{get, post}; use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; -use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{ + location_health, location_work, run_reads, runs, snapshots, trigger_reads, +}; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; use crate::store::errors::ControlError; @@ -19,6 +21,14 @@ pub async fn serve_api( ) -> Result<(), ControlError> { migrate_store(&store).await?; let protected = Router::new() + .route( + "/api/locations/{location_id}/work/poll", + post(location_work::poll), + ) + .route( + "/api/locations/{location_id}/work/{request_id}", + get(location_work::result).post(location_work::complete), + ) .route( "/api/locations/{location_id}/health", get(location_health::recent).post(location_health::report), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 5e4837fa3b..d5070b7bb9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,12 +2,17 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{location_health, run_reads, runs, snapshots, trigger_reads}; +use crate::api::_helpers::{ + location_health, location_work, run_reads, runs, snapshots, trigger_reads, +}; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( location_health::report, location_health::recent, + location_work::poll, + location_work::complete, + location_work::result, snapshots::publish, snapshots::current, runs::execution, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs new file mode 100644 index 0000000000..c523d62210 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs @@ -0,0 +1,40 @@ +//! Request retries and completion share the same immutable result reader and instance scope. + +use uuid::Uuid; + +use crate::platform::models::StoredLocationWork; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_work( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ + w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ + FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ + WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +pub(crate) async fn require_instance( + transaction: ControlTransaction, + (location_id, session_id, target): (Uuid, Uuid, &str), +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT i.instance_id FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE i.instance_id=$1 AND i.location_id=$2 AND i.target=$3 AND l.archived_at IS NULL", + vec![Parameter::Id(session_id), Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "a registered instance of this location and target is required".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index e32efd2a5f..8f5f6285fa 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod location_heartbeat; +pub(crate) mod location_work; pub(crate) mod observations; pub(crate) mod watchdog; +pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs new file mode 100644 index 0000000000..fe1e104915 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -0,0 +1,51 @@ +//! The authenticated client verifies every leased request's location, target and instance. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::models::MutationResponse; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, +}; + +impl LocationWorkClient { + pub async fn poll( + &self, + poll: &LocationWorkPoll, + ) -> Result, ClientError> { + let claim: Option = client_response( + self.connection + .client + .post(format!("{}poll", self.work_url)) + .json(poll), + ) + .await?; + if claim.as_ref().is_some_and(|claim| { + claim.request.location_id != self.location_id + || claim.request.target != poll.target + || claim.session_id != poll.session_id + }) { + return Err(ClientError::Configuration( + "work response does not match this location, target and instance", + )); + } + Ok(claim) + } + + /// Retain the same request, claim and result when retrying an uncertain acknowledgement. + pub async fn complete( + &self, + request_id: Uuid, + result: &LocationWorkResult, + ) -> Result { + let response: MutationResponse = client_response( + self.connection + .client + .post(format!("{}{request_id}", self.work_url)) + .json(result), + ) + .await?; + Ok(response.applied) + } +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs new file mode 100644 index 0000000000..fd24042528 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -0,0 +1,63 @@ +//! Serialized leases fence results from replaced location instances and uncertain poll retries. + +use chrono::Duration; +use uuid::Uuid; + +use crate::platform::_helpers::location_work::{read_work, require_instance}; +use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn claim_location_work( + store: &ControlStore, + location_id: Uuid, + poll: &LocationWorkPoll, +) -> Result, ControlError> { + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, poll.session_id, &poll.target), + ) + .await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.expires_at>$3 AND NOT EXISTS (SELECT 1 FROM sqb_location_work_results r WHERE r.request_id=w.request_id) \ + AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ + ORDER BY w.requested_at, w.request_id LIMIT 1", + vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; + let Some(row) = rows.first() else { + commit(transaction).await?; + return Ok(None); + }; + let request_id = serde_json::from_value(row["request_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; + let (transaction, stored) = read_work(transaction, request_id).await?; + let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; + let retained = stored.session_id == Some(poll.session_id) + && stored.claimed_until.is_some_and(|until| until > now); + let (claim_id, lease_expires_at) = if retained { + ( + stored.claim_id.ok_or(ControlError::Fenced)?, + stored.claimed_until.ok_or(ControlError::Fenced)?, + ) + } else { + ( + Uuid::now_v7(), + (now + Duration::seconds(60)).min(stored.record.expires_at), + ) + }; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET claim_id=$1, session_id=$2, claimed_until=$3 WHERE request_id=$4", + vec![Parameter::Id(claim_id), Parameter::Id(poll.session_id), Parameter::Time(lease_expires_at), Parameter::Id(request_id)]).await?; + commit(transaction).await?; + Ok(Some(LocationWorkClaim { + request: stored.record.request_json, + generation: stored.record.generation, + claim_id, + session_id: poll.session_id, + lease_expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs new file mode 100644 index 0000000000..0f1888ed8b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -0,0 +1,57 @@ +//! A completion is immutable; only the current unexpired claim may publish its result. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::_helpers::location_work::{read_work, require_instance}; +use crate::platform::models::LocationWorkResult; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn complete_location_work( + store: &ControlStore, + (location_id, request_id): (Uuid, Uuid), + result: &LocationWorkResult, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, stored) = read_work(transaction, request_id).await?; + let stored = stored + .filter(|stored| stored.record.request_json.location_id == location_id) + .ok_or_else(|| ControlError::Invalid("work is not assigned to this location".into()))?; + let transaction = require_instance( + transaction, + ( + location_id, + result.session_id, + &stored.record.request_json.target, + ), + ) + .await?; + if let Some(previous) = stored.record.result_json { + if previous != *result { + return Err(ControlError::Invalid( + "location work already has a different result".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + if stored.claim_id != Some(result.claim_id) + || stored.session_id != Some(result.session_id) + || stored + .claimed_until + .is_none_or(|until| until <= transaction.now) + || stored.record.expires_at <= transaction.now + { + return Err(ControlError::Fenced); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work_results (request_id,result_json,completed_at) VALUES ($1,$2,$3)", + vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs new file mode 100644 index 0000000000..a0df47b88d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs @@ -0,0 +1,21 @@ +//! Reuse one credential-bearing pool for the location's outbound work channel. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::connect_client::connect_client; +use crate::platform::models::LocationWorkClient; + +pub fn connect_location_work( + (server_url, bearer): (&str, &str), + location_id: Uuid, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let work_url = + connection.endpoint(&["api", "locations", &location_id.to_string(), "work", ""])?; + Ok(LocationWorkClient { + connection, + work_url, + location_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs new file mode 100644 index 0000000000..f71f848d79 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs @@ -0,0 +1,20 @@ +//! Read one request and its durable result without persisting a derived pending/expired status. + +use uuid::Uuid; + +use crate::platform::_helpers::location_work::read_work; +use crate::platform::models::LocationWorkRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn location_work( + store: &ControlStore, + request_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, record) = read_work(transaction, request_id).await?; + commit(transaction).await?; + Ok(record.map(|stored| stored.record)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index a633f3e7ce..095c42262e 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,4 +1,9 @@ +pub mod claim_location_work; +pub mod complete_location_work; +pub mod connect_location_work; pub mod location_health; +pub mod location_work; pub mod report_location_health; pub mod report_remote_location_health; +pub mod request_location_work; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs new file mode 100644 index 0000000000..d03b92dce4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -0,0 +1,81 @@ +//! Only trusted store owners enqueue named work; retries preserve the original generation/deadline. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::platform::_helpers::location_work::read_work; +use crate::platform::models::LocationWorkRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn request_location_work( + store: &ControlStore, + context: &AuditContext, + request: &LocationWorkRequest, +) -> Result { + if request.target.trim().is_empty() + || request.reason.trim().is_empty() + || context.actor.trim().is_empty() + { + return Err(ControlError::Invalid( + "location work needs a target, reason and trusted actor".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = read_work(transaction, request.request_id).await?; + if let Some(previous) = previous { + if previous.record.request_json != *request || previous.record.requested_by != context.actor + { + return Err(ControlError::Invalid( + "location work ID already has different content or actor".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ + JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ + WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; + let snapshot = rows + .first() + .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; + let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; + let generation = snapshot["generation"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + Duration::minutes(5))]).await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&request.request_id, b"location-work-requested"), + "location.work_requested", + ), + ( + json!({"location_id": request.location_id, "target": request.target}), + json!(null), + json!(request), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 052d9bf775..29f902dff0 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -4,8 +4,17 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; +/// One pooled outbound connection; the location never receives a store handle. +#[derive(Clone, Debug)] +pub struct LocationWorkClient { + pub(crate) connection: ControlClient, + pub(crate) work_url: reqwest::Url, + pub(crate) location_id: Uuid, +} + /// Locations report bounded facts, never connection strings or user exception text. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] @@ -31,6 +40,76 @@ pub struct LocationHealthObservation { pub report: LocationHealthReport, } +/// Only named location operations cross this boundary, never arbitrary SQL or Python. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWork { + CheckConnection {}, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkRequest { + pub request_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub reason: String, + pub work: LocationWork, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkPoll { + pub session_id: Uuid, + pub target: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkClaim { + pub request: LocationWorkRequest, + pub generation: i64, + pub claim_id: Uuid, + pub session_id: Uuid, + pub lease_expires_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWorkOutcome { + Connection { healthy: bool }, + Superseded {}, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkResult { + pub claim_id: Uuid, + pub session_id: Uuid, + pub outcome: LocationWorkOutcome, +} + +/// Expiry is derived from the deadline; only requests, leases and completed results are stored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationWorkRecord { + pub request_json: LocationWorkRequest, + pub requested_by: String, + pub requested_at: DateTime, + pub expires_at: DateTime, + pub generation: i64, + pub result_json: Option, + pub completed_at: Option>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredLocationWork { + #[serde(flatten)] + pub(crate) record: LocationWorkRecord, + pub(crate) claim_id: Option, + pub(crate) session_id: Option, + pub(crate) claimed_until: Option>, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index e2933cdb64..cd44c4e59a 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -8,6 +8,7 @@ use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); +pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/location_work.sql b/crates/sqlbuild-orchestration/src/store/location_work.sql new file mode 100644 index 0000000000..2f5274b305 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/location_work.sql @@ -0,0 +1,20 @@ +CREATE TABLE sqb_location_work ( + request_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json {json} NOT NULL, + requested_by TEXT NOT NULL, + requested_at {time} NOT NULL, + expires_at {time} NOT NULL, + claim_id {uuid}, + session_id {uuid} REFERENCES sqb_location_instances(instance_id), + claimed_until {time} +); +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); +CREATE TABLE sqb_location_work_results ( + request_id {uuid} PRIMARY KEY REFERENCES sqb_location_work(request_id), + result_json {json} NOT NULL, + completed_at {time} NOT NULL +); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 9063a5fedd..31fefb1c4e 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -3,7 +3,9 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::store::_helpers::migration::{LOCATION_INSTANCES, SCHEMA, render, verify_migrations}; +use crate::store::_helpers::migration::{ + LOCATION_INSTANCES, LOCATION_WORK, SCHEMA, render, verify_migrations, +}; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -25,8 +27,12 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { .await?; let recall_statements = recall_schema(store); let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - let location_digest = sha256_hex(LOCATION_INSTANCES.as_bytes()); - let digests = [digest.clone(), location_digest.clone()]; + let additions = [ + ("location_instances", LOCATION_INSTANCES), + ("location_work", LOCATION_WORK), + ]; + let mut digests = vec![digest.clone()]; + digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); let current = verify_migrations(&rows, &digests)?; if rows.is_empty() { for statement in SCHEMA @@ -48,8 +54,11 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { VALUES (1, 'control_store', $1, $2, $3)", vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } - if rows.len() < digests.len() { - for statement in LOCATION_INSTANCES + for (index, (name, sql)) in additions.into_iter().enumerate() { + if index + 1 < rows.len() { + continue; + } + for statement in sql .split(';') .map(str::trim) .filter(|statement| !statement.is_empty()) @@ -59,8 +68,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { let now = transaction.now; (transaction, _) = query(transaction, "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ - VALUES (2, 'location_instances', $1, $2, $3)", - vec![Parameter::Text(location_digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + VALUES ($1, $2, $3, $4, $5)", + vec![Parameter::Integer(index as i64 + 2), Parameter::Text(name.into()), + Parameter::Text(digests[index + 1].clone()), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs index 55a6bce444..0b7016321c 100644 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ b/crates/sqlbuild-orchestration/tests/api.rs @@ -16,9 +16,13 @@ mod store_test_types; mod test_contract; #[path = "api/test_location_health.rs"] mod test_location_health; +#[path = "api/test_location_work.rs"] +mod test_location_work; #[path = "api/test_reads.rs"] mod test_reads; #[path = "api/test_types.rs"] mod test_types; #[path = "api/test_worker.rs"] mod test_worker; +#[path = "api/work/helpers.rs"] +mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs new file mode 100644 index 0000000000..52b883742f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -0,0 +1,302 @@ +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; +use sqlbuild_orchestration::platform::main::location_work::location_work; +use sqlbuild_orchestration::platform::main::report_location_health::report_location_health; +use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkRecord, LocationWorkRequest, LocationWorkResult, +}; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::helpers::{start_api, stop_api}; +use crate::location_helpers::create_location; +use crate::run_helpers::snapshot; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::LocationWorkTestCase; +use crate::work_helpers::execute; + +#[test] +fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable_results_are_enforced() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite outbound work", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL outbound work", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("fresh migration"); + // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version=3").await; + migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + migrate_store(store).await.expect("schema 1 upgrade"); + migrate_store(store).await.expect("repeat migration"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let request = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id, + target: "preview #1".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + assert!( + request_location_work(store, &audit, &request) + .await + .is_err(), + "unpublished target" + ); + let mut input = snapshot(location_id); + input.target.clone_from(&request.target); + publish_snapshot(store, &input).await.expect("publish"); + let invalid_audit = AuditContext { + granting_role: String::new(), + ..audit.clone() + }; + assert!( + request_location_work(store, &invalid_audit, &request) + .await + .is_err() + ); + assert!( + location_work(store, request.request_id) + .await + .expect("rolled back read") + .is_none() + ); + assert!( + request_location_work(store, &audit, &request) + .await + .expect("enqueue") + ); + let original = location_work(store, request.request_id) + .await + .expect("read") + .expect("request"); + input.generation = 2; + publish_snapshot(store, &input).await.expect("republish"); + assert!( + !request_location_work(store, &audit, &request) + .await + .expect("retry after redeploy") + ); + assert_eq!( + location_work(store, request.request_id) + .await + .expect("read"), + Some(original.clone()) + ); + let changed = LocationWorkRequest { + reason: "Different request".into(), + ..request.clone() + }; + assert!( + request_location_work(store, &audit, &changed) + .await + .is_err() + ); + let changed_actor = AuditContext { + actor: "another-admin".into(), + ..audit.clone() + }; + assert!( + request_location_work(store, &changed_actor, &request) + .await + .is_err() + ); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let worker = + connect_location_work((&api.url, &token.bearer), location_id).expect("work client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: request.target.clone(), + }; + assert!(worker.poll(&poll).await.is_err(), "register before polling"); + for session_id in [poll.session_id, Uuid::nil()] { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register instance"); + } + let wrong_target = LocationWorkPoll { + target: "another target".into(), + ..poll.clone() + }; + assert!(worker.poll(&wrong_target).await.is_err()); + let foreign = + connect_location_work((&api.url, &token.bearer), other).expect("foreign client"); + assert!(matches!( + foreign.poll(&poll).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/work", api.url); + let response = client + .post(format!("{url}/poll")) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let claim = worker.poll(&poll).await.expect("poll").expect("claim"); + assert_eq!( + claim.generation, test_case.expected_generation, + "{}: request remains pinned despite redeployment", + test_case.description + ); + assert_eq!(claim.request, request); + assert_eq!( + worker.poll(&poll).await.expect("lost poll response retry"), + Some(claim.clone()) + ); + let replacement = LocationWorkPoll { + session_id: Uuid::nil(), + ..poll.clone() + }; + assert!( + worker + .poll(&replacement) + .await + .expect("another instance") + .is_none() + ); + let stale = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: false }, + }; + execute( + store, + "UPDATE sqb_location_work SET claimed_until='2000-01-01T00:00:00Z'", + ) + .await; + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let reclaimed = worker + .poll(&replacement) + .await + .expect("reclaim") + .expect("new claim"); + assert_ne!(claim.claim_id, reclaimed.claim_id); + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let result = LocationWorkResult { + claim_id: reclaimed.claim_id, + session_id: replacement.session_id, + outcome: LocationWorkOutcome::Superseded {}, + }; + assert!( + worker + .complete(request.request_id, &result) + .await + .expect("complete") + ); + execute(store, "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z', claimed_until='2000-01-01T00:00:00Z'").await; + assert!( + !worker + .complete(request.request_id, &result) + .await + .expect("lost acknowledgement retry after expiry") + ); + let changed_result = LocationWorkResult { + outcome: LocationWorkOutcome::Connection { healthy: true }, + ..result.clone() + }; + assert!( + worker + .complete(request.request_id, &changed_result) + .await + .is_err() + ); + assert!( + worker + .poll(&poll) + .await + .expect("completed excluded") + .is_none() + ); + let response = client + .get(format!("{url}/{}", request.request_id)) + .bearer_auth(&token.bearer) + .send() + .await + .expect("result read"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let record: LocationWorkRecord = response.json().await.expect("typed result"); + assert_eq!(record.result_json, Some(result)); + assert!(record.completed_at.is_some()); + let next = LocationWorkRequest { + request_id: Uuid::now_v7(), + ..request + }; + request_location_work(store, &audit, &next) + .await + .expect("another request"); + execute( + store, + "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z'", + ) + .await; + assert!( + worker + .poll(&poll) + .await + .expect("expired excluded") + .is_none() + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 59c0610602..cf088cb040 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -39,3 +39,9 @@ pub(crate) struct LocationHealthTestCase { pub(crate) backend: Backend, pub(crate) expected_observations: usize, } + +pub(crate) struct LocationWorkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_generation: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs new file mode 100644 index 0000000000..32a7a3cbd3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -0,0 +1,20 @@ +//! Fault injection changes only disposable test stores, never product time or lease semantics. + +use sqlbuild_orchestration::store::models::ControlStore; + +pub(crate) async fn execute(store: &ControlStore, sql: &str) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(sql) + .expect("inject SQLite fault"); + return; + }; + sqlx_core::raw_sql::raw_sql(sql) + .execute(pool) + .await + .expect("inject PostgreSQL fault"); +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index ab2594e4e4..e486bb4182 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version=2", + "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ + DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", ) .expect("reconstruct the previous schema without changing existing facts"); migrate_store(&store) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index eebedd8538..66bcebbc27 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -58,3 +58,10 @@ class LocationDatabaseTestCase: database: str expected_generation: int connection_toml: str + + +@dataclass(frozen=True) +class LocationWorkTestCase: + description: str + database: str + expected_connection: bool diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index efa304cafd..d10a50b37c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -120,6 +120,25 @@ def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: P return int(pidfile.read_text()) +def wait_for_location_work( + api: SnapshotApi, request_id: str, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + """Observe completion through the scoped API rather than private coordination tables.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/work/{request_id}", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 15 + result: dict[str, Any] = {"result_json": None} + while result["result_json"] is None: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + result = json.load(response) + time.sleep(0.02) + return result + + def api_address(process: subprocess.Popen[bytes], log: Path) -> str: """Wait for bounded native startup, never publishing an unready address.""" deadline: float = time.monotonic() + 10 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py new file mode 100644 index 0000000000..502613c572 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py @@ -0,0 +1,158 @@ +"""Trusted owner requests cross the real HTTP channel to an isolated DuckDB connection check.""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationWorkTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_location_health, + wait_for_location_work, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationWorkTestCase("healthy requested check", "warehouse.db", True), + LocationWorkTestCase("failed requested check", "missing/warehouse.db", False), + ], + ids=lambda case: case.description, +) +def test_given_a_published_location_when_an_owner_requests_work_then_results_return_outbound_and_stale_work_is_not_executed( + test_case: LocationWorkTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + blocked: Path = tmp_path / "block-debug" + child_pid: Path = tmp_path / "debug.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + '\n[targets."preview #1"]\nschema = "main"\n', + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if 'debug' in sys.argv and Path({str(blocked)!r}).exists():\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + f" while Path({str(blocked)!r}).exists():\n" + " time.sleep(0.02)\n" + ), + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # The serving location must never open the owner's database. + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + owner_environment: dict[str, str] = os.environ | { + "SQLBUILD_CONTROL_DATABASE_URL": str(tmp_path / "control/.sqlbuild/control.sqlite3") + } + arguments: list[str] = [ + executable, + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "preview #1", + ] + request_id: str = str(uuid4()) + enqueue: list[str] = [ + executable, + "admin", + "check-location", + api.location_id, + "--target", + "preview #1", + "--request-id", + request_id, + "--reason", + "Check orders connectivity", + ] + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 1, process, log) + _ = wait_for_location_health( + api, + "preview #1", + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + first: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(first.stdout) == {"request_id": request_id, "applied": True} + retry: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(retry.stdout) == {"request_id": request_id, "applied": False} + result: dict[str, Any] = wait_for_location_work(api, request_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == { + "kind": "connection", + "healthy": test_case.expected_connection, + } + read: subprocess.CompletedProcess[str] = subprocess.run( + [executable, "admin", "location-work", request_id], + env=owner_environment, + capture_output=True, + text=True, + check=True, + ) + assert json.loads(read.stdout) == result + process.terminate() + _ = process.wait(timeout=5) + stale_id: str = str(uuid4()) + enqueue[enqueue.index(request_id)] = stale_id + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 2, process, log) + result = wait_for_location_work(api, stale_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + changed_id: str = str(uuid4()) + enqueue[enqueue.index(stale_id)] = changed_id + blocked.touch() + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + _ = wait_for_interpreter(process, child_pid, log) + _ = (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(17 AS INTEGER)" + ) + ) + blocked.unlink() + result = wait_for_location_work(api, changed_id, process, log) + assert result["generation"] == 2 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + _ = wait_for_generation(api, "preview #1", 3, process, log) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() From d9657054df50b0500501e2e241c3c582a7f59d42 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:53:04 +0000 Subject: [PATCH 034/138] feat: execute scheduled jobs with supervised local workers --- Cargo.toml | 2 +- .../rpt_edge_scheduled_function.sql | 7 + .../sqlbuild-cli/src/argv/command_table.json | 30 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- crates/sqlbuild-cli/src/lib.rs | 1 + .../src/native_commands/_helpers/execution.rs | 14 +- .../src/native_commands/main/_run_control.rs | 9 +- .../src/workers/_helpers/configuration.rs | 40 +++ .../src/workers/_helpers/local.rs | 296 ++++++++++++++++++ .../sqlbuild-cli/src/workers/_helpers/mod.rs | 3 + .../src/workers/_helpers/shutdown.rs | 148 +++++++++ crates/sqlbuild-cli/src/workers/main/mod.rs | 1 + .../src/workers/main/open_local_workers.rs | 18 ++ crates/sqlbuild-cli/src/workers/mod.rs | 5 + crates/sqlbuild-cli/src/workers/models.rs | 47 +++ .../tests/native_commands/worker/helpers.rs | 13 +- .../src/api/_helpers/authentication.rs | 41 ++- .../src/api/_helpers/run_reads.rs | 99 +++++- .../src/api/_helpers/trigger_reads.rs | 49 +-- .../src/api/main/serve_api.rs | 8 + .../sqlbuild-orchestration/src/api/models.rs | 23 ++ .../src/runs/main/claim_next_run.rs | 6 +- .../src/runs/main/job_write_set.rs | 2 +- .../src/runs/main/mod.rs | 1 + .../src/runs/main/observe_run.rs | 24 ++ .../sqlbuild-orchestration/src/runs/models.rs | 10 + .../src/scheduler/main/run_scheduler.rs | 29 +- .../src/scheduler/mod.rs | 2 +- .../src/scheduler/types.rs | 19 ++ .../src/store/main/list_recall_invocations.rs | 62 ++++ .../src/store/main/mod.rs | 1 + .../tests/api/test_contract.rs | 2 +- .../tests/api/test_reads.rs | 166 +++++++++- .../tests/api/test_worker.rs | 4 +- .../tests/credentials/test_contract.rs | 2 +- .../tests/runs/monitor/helpers.rs | 2 +- .../tests/runs/test_claims.rs | 124 +++++++- .../tests/runs/test_coalesce.rs | 2 +- .../tests/runs/test_faults.rs | 8 +- .../tests/runs/test_lifecycle.rs | 10 +- .../tests/runs/test_monitor.rs | 4 +- .../tests/runs/test_requests.rs | 2 +- .../tests/runs/test_types.rs | 7 + .../tests/scheduler/ticks/test_loop.rs | 9 +- .../tests/scheduler/ticks/test_types.rs | 20 ++ .../_helpers/boundary/config_values.rs | 32 ++ .../_helpers/build_command/command.rs | 28 +- .../_helpers/build_command/execution.rs | 4 + .../_helpers/build_command/invocation.rs | 4 + .../build_progress/build_callbacks.rs | 22 +- .../src/bindings/_helpers/cli/dispatch.rs | 15 +- .../src/bindings/_helpers/cli/entry.rs | 44 ++- .../_helpers/compiled_project/constants.rs | 5 + .../bindings/_helpers/compiled_project/mod.rs | 2 + .../compiled_project/snapshot_execution.rs | 247 +++++++++++++++ .../compiled_project/snapshot_worker.rs | 58 +++- .../_helpers/location_command/command.rs | 47 ++- .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/worker.rs | 154 +++++++++ .../src/bindings/_helpers/recall/session.rs | 106 +++++-- .../bindings/_helpers/rules/rules_config.rs | 28 +- .../publication/_test_types.py | 27 +- .../compiled_project/publication/helpers.py | 188 ++++++++++- .../publication/test_worker.py | 275 ++++++++++++++++ 64 files changed, 2463 insertions(+), 200 deletions(-) create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/configuration.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/local.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs create mode 100644 crates/sqlbuild-cli/src/workers/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/workers/main/open_local_workers.rs create mode 100644 crates/sqlbuild-cli/src/workers/mod.rs create mode 100644 crates/sqlbuild-cli/src/workers/models.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/observe_run.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs diff --git a/Cargo.toml b/Cargo.toml index 01ca1f28c3..61061e23b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,7 +115,7 @@ errno = "0.3" windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std", "termios"] } saphyr-parser = "0.1" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..5e33820e53 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql @@ -0,0 +1,7 @@ +MODEL ( + description 'A scheduled model whose implicit scalar-function writer must be claimed.', + tags [edge_case], + materialized table, +); + +SELECT __udf("edge_next_quantity")(CAST(23 AS INTEGER)) AS quantity diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d86b9c0ac6..08bbcf87c6 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -106,7 +106,7 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish", "serve", "runtime"], + "choices": ["publish", "serve", "runtime", "execute"], "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", "group": 0 }, @@ -149,6 +149,20 @@ "help": "immutable deployment image identity to pin for workers", "group": 1 }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "location_run_id", + "help": "frozen run UUID for execute; requires SQLBUILD_RUN_TOKEN", + "group": 1 + }, + { + "kind": "store", + "options": ["--claim-id"], + "dest": "location_claim_id", + "help": "active claim UUID for execute", + "group": 1 + }, { "kind": "store", "options": ["--project-dir", "--sqb-project-dir"], @@ -849,6 +863,13 @@ "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", "group": 1 }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 + }, { "kind": "store", "options": [ @@ -934,6 +955,13 @@ "default": "SQLBUILD_CONTROL_DATABASE_URL", "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", "group": 1 + }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 } ] } diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 2f6ce26c7f..247242fd45 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,7 +515,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const LOCATION_ARGUMENTS: [Field; 7] = [ +const LOCATION_ARGUMENTS: [Field; 9] = [ field("project_dir", Source::EffectiveProjectDir), SELECTED_TARGET, same("location_id"), @@ -523,6 +523,8 @@ const LOCATION_ARGUMENTS: [Field; 7] = [ attr("generation", "location_generation"), attr("image_id", "location_image_id"), attr("action", "location_command"), + attr("run_id", "location_run_id"), + attr("claim_id", "location_claim_id"), ]; pub(crate) const LOCATION_ROUTE: Route = routed( "run_location", diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 538dec3263..5364b9c483 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -10,3 +10,4 @@ pub mod dispatch; pub mod launcher; pub mod native_commands; pub mod replay; +pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index a9d41344e3..41b1df3e3f 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -29,7 +29,7 @@ use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; -use crate::native_commands::_helpers::decision::{control_database, text}; +use crate::native_commands::_helpers::decision::{control_database, flag, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::{location_database_files, project_location}; @@ -48,6 +48,8 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +use crate::workers::main::open_local_workers::open_local_workers; +use crate::workers::models::LocalWorkers; /// Request one publication without transferring interpreter ownership to a cancellable future. pub(crate) async fn compile_generation( @@ -311,6 +313,16 @@ pub(crate) async fn open_control_store( Ok(store) } +/// Decode privileged CLI configuration before handing process ownership to the workers domain. +pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { + open_local_workers( + text(namespace, "control_launchers").as_deref(), + text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), + !flag(namespace, "control_no_scheduler"), + ) +} + pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { #[cfg(unix)] { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 1f39c05c8b..1ca4f4f3b1 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -10,7 +10,9 @@ use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{control_address, flag}; -use crate::native_commands::_helpers::execution::{open_control_store, supervise_control_roles}; +use crate::native_commands::_helpers::execution::{ + open_control_store, open_control_workers, supervise_control_roles, +}; use crate::native_commands::_helpers::terminal::write_output; async fn stopped(mut shutdown: watch::Receiver) { @@ -27,6 +29,7 @@ pub(crate) async fn run_control( ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; let store = open_control_store(namespace).await?; + let workers = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); let (_changes, changes) = watch::channel(()); let mut tasks = tokio::task::JoinSet::new(); @@ -66,8 +69,8 @@ pub(crate) async fn run_control( run_scheduler( &scheduler_store, SchedulerState::new(Uuid::now_v7()), - shutdown, - changes, + (shutdown, changes), + workers, ) .await }); diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs new file mode 100644 index 0000000000..bf486eeb40 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -0,0 +1,40 @@ +//! Reject ambiguous or nonexistent installation-owned launcher scopes before claiming any run. + +use std::collections::BTreeSet; + +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::workers::models::LocalLaunchers; + +pub(in crate::workers) fn read_configuration( + path: Option<&str>, + scheduler_enabled: bool, +) -> Result { + let configuration: LocalLaunchers = match path { + Some(path) => serde_json::from_slice(&std::fs::read(path)?) + .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, + None => LocalLaunchers { + local: Vec::new(), + concurrency_limits: Default::default(), + }, + }; + if !scheduler_enabled && !configuration.local.is_empty() { + return Err(ControlError::Invalid( + "launchers require the embedded scheduler".into(), + )); + } + let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); + for location in &configuration.local { + if location.target.trim().is_empty() + || !location.project_dir.is_absolute() + || !location.project_dir.is_dir() + || !location.executable.is_absolute() + || !location.executable.is_file() + || !scopes.insert((location.location_id, location.target.as_str())) + { + return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); + } + } + Ok(configuration) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/local.rs new file mode 100644 index 0000000000..49bbbe75af --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/local.rs @@ -0,0 +1,296 @@ +//! The privileged control role owns and reaps local children; workers receive claim-scoped HTTP. + +use std::io::Write; +use std::process::{ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use chrono::{TimeDelta, Utc}; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::process::{Child, Command}; +use uuid::Uuid; + +use crate::workers::models::{LocalLauncher, LocalWorker, LocalWorkers}; + +impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { + async fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + for index in (0..self.active.len()).rev() { + let worker = &mut self.active[index]; + let mut state = observe_run(store, worker.claim.request.run_id).await?; + if let Some(child) = worker.child.as_mut() { + worker.exit = worker.exit.or(child.try_wait()?); + } + if worker.exit.is_none() + && let Some(child) = worker.child.as_mut() + { + let now = state.observed_at; + let not_started = matches!(state.status, RunStatus::Queued | RunStatus::Launched) + && state + .claimed_at + .is_some_and(|claimed| now - claimed > TimeDelta::minutes(2)); + let silent = state + .last_heartbeat_at + .is_some_and(|last| now - last > TimeDelta::minutes(10)); + if (state.cancel_requested_at.is_some() || not_started || silent) + && worker.stopping.is_none() + { + terminate(child)?; + worker.stopping = Some(Instant::now()); + } + if worker + .stopping + .is_some_and(|at| at.elapsed() >= Duration::from_secs(60)) + { + child.kill().await?; + worker.exit = child.try_wait()?; + state = observe_run(store, worker.claim.request.run_id).await?; + } + } + if worker.exit.is_some() || worker.child.is_none() { + if !terminal(state.status) { + let Some(lease) = lease else { + continue; + }; + let outcome = if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }; + publish( + store, + lease, + &worker.claim, + ( + "local-exit", + RunChange::Finished { + outcome, + reason: "local worker exited; physical child has been reaped" + .into(), + exit_code: worker.exit.map(exit_code), + }, + ), + ) + .await?; + } + self.active.swap_remove(index); + } + } + let Some(lease) = lease else { + return Ok(()); + }; + for location in &self.configuration.local { + let running = self + .active + .iter() + .filter(|worker| { + worker.claim.request.location_id == location.location_id + && worker.claim.request.target == location.target + }) + .count(); + for _ in running..location.max_workers.get() { + let Some(claim) = claim_next_run( + store, + lease, + &self.configuration.concurrency_limits, + Some((location.location_id, &location.target)), + ) + .await? + else { + break; + }; + let index = self.active.len(); + self.active.push(LocalWorker { + claim, + child: None, + stopping: None, + exit: None, + }); + let worker = &mut self.active[index]; + let claim = &worker.claim; + if claim.request.image_id.is_some() { + publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; + self.active.swap_remove(index); + continue; + } + let context = AuditContext { + actor: format!("scheduler/{}", lease.holder), + granting_role: "scheduler".into(), + request_id: claim.claim_id, + ip_address: None, + }; + let credential = issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + Utc::now() + TimeDelta::days(7), + ) + .await?; + publish( + store, + lease, + claim, + ("local-launched", RunChange::Launched { container_id: None }), + ) + .await?; + match spawn(location, claim, &credential.bearer, &self.control_variable) { + Ok(child) => { + drop(writeln!( + std::io::stderr().lock(), + "Worker launched for run {}", + claim.request.run_id + )); + worker.child = Some(child); + } + Err(_) => { + publish( + store, + lease, + claim, + ( + "local-spawn-failed", + RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "local worker process could not start".into(), + exit_code: None, + }, + ), + ) + .await?; + self.active.swap_remove(index); + } + } + } + } + Ok(()) + } + + /// Never release claims until children have physically stopped, even if shutdown loses its fence. + async fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.shutdown_workers(store, lease).await + } +} + +fn spawn( + location: &LocalLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, +) -> Result { + Ok(Command::new(&location.executable) + .args([ + "location", + "execute", + "--server", + &location.server_url, + "--location-id", + &location.location_id.to_string(), + "--target", + &location.target, + "--run-id", + &claim.request.run_id.to_string(), + "--claim-id", + &claim.claim_id.to_string(), + "--project-dir", + ]) + .arg(&location.project_dir) + .current_dir(&location.project_dir) + .env_remove(control_variable) + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .env_remove("SQLBUILD_LOCATION_TOKEN") + .env("SQLBUILD_RUN_TOKEN", bearer) + .env( + INVOCATION_CONTEXT_ENV, + serde_json::json!({ + "run_id": claim.request.run_id, + "integration": { + "name": "sqlbuild", + "run_id": claim.request.run_id, + "job_name": claim.request.job_key, + }, + }) + .to_string(), + ) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn()?) +} + +pub(in crate::workers) async fn publish( + store: &ControlStore, + lease: &SchedulerLease, + claim: &RunClaim, + (event, change): (&str, RunChange), +) -> Result<(), ControlError> { + change_run( + store, + RunAuthority::Scheduler(lease), + &RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, event.as_bytes()), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await?; + Ok(()) +} + +pub(in crate::workers) fn terminal(status: RunStatus) -> bool { + !matches!( + status, + RunStatus::Queued | RunStatus::Launched | RunStatus::Running | RunStatus::Unresponsive + ) +} + +pub(in crate::workers) fn exit_code(exit: ExitStatus) -> i64 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + i64::from( + exit.code() + .unwrap_or_else(|| 128 + exit.signal().unwrap_or(0)), + ) + } + #[cfg(not(unix))] + i64::from(exit.code().unwrap_or(1)) +} + +pub(in crate::workers) fn terminate(child: &mut Child) -> Result<(), ControlError> { + #[cfg(unix)] + if let Some(id) = child.id() { + let raw = i32::try_from(id) + .map_err(|error| ControlError::Invalid(format!("invalid child process ID: {error}")))?; + let pid = rustix::process::Pid::from_raw(raw) + .ok_or_else(|| ControlError::Invalid("invalid child process ID".into()))?; + match rustix::process::kill_process(pid, rustix::process::Signal::TERM) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + #[cfg(not(unix))] + let _ = child; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs new file mode 100644 index 0000000000..16f45af3e7 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(super) mod configuration; +mod local; +mod shutdown; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs new file mode 100644 index 0000000000..24ab13b5cf --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -0,0 +1,148 @@ +//! Keep leadership alive while reaping children, then release only successfully reported claims. + +use std::io::Write; +use std::time::{Duration, Instant}; + +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{RunChange, RunOutcome}; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::_helpers::local::{exit_code, publish, terminal, terminate}; +use crate::workers::models::{LocalWorker, LocalWorkers}; + +impl LocalWorker { + async fn reap( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let Some(child) = self.child.as_mut() else { + return Ok(()); + }; + if self.exit.is_some() { + return Ok(()); + } + if self.stopping.is_none() { + if terminate(child).is_err() { + child.kill().await?; + } + self.stopping = Some(Instant::now()); + } + let deadline = self.stopping.unwrap_or_else(Instant::now) + Duration::from_secs(60); + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + loop { + tokio::select! { + exit = child.wait() => { + self.exit = Some(exit?); + return Ok(()); + } + _ = tokio::time::sleep_until(deadline.into()) => { + child.kill().await?; + self.exit = Some(child.wait().await?); + return Ok(()); + } + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && renew_lease(store, lease).await.is_err() + { + drop(writeln!(std::io::stderr().lock(), "Scheduler lease renewal failed during worker shutdown")); + } + } + } + } + } + + async fn finish_shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.reap(store, lease).await?; + let state = observe_run(store, self.claim.request.run_id).await?; + if !terminal(state.status) { + let lease = lease.ok_or(ControlError::Fenced)?; + let reason = if self.child.is_none() { + "local worker launch preparation failed before a child started" + } else { + "scheduler stopped its local worker and reaped the child" + }; + publish( + store, + lease, + &self.claim, + ( + "local-shutdown", + RunChange::Finished { + outcome: if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }, + reason: reason.into(), + exit_code: self.exit.map(exit_code), + }, + ), + ) + .await?; + } + Ok(()) + } +} + +impl LocalWorkers { + pub(in crate::workers) async fn shutdown_workers( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let mut retrying = false; + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + while !self.active.is_empty() { + let mut failure = None; + for index in (0..self.active.len()).rev() { + match self.active[index].finish_shutdown(store, lease).await { + Ok(()) => { + self.active.swap_remove(index); + } + Err(ControlError::Fenced) => failure = Some(ControlError::Fenced), + Err(error) => { + failure.get_or_insert(error); + } + } + } + if matches!(failure, Some(ControlError::Fenced)) { + return Err(ControlError::Fenced); + } + if failure.is_some() { + if !retrying { + drop(writeln!( + std::io::stderr().lock(), + "Retrying worker shutdown publication; stopped workers and their claims remain owned" + )); + retrying = true; + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(1)) => {}, + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && matches!(renew_lease(store, lease).await, Err(ControlError::Fenced)) + { + return Err(ControlError::Fenced); + } + } + } + } + } + if retrying { + drop(writeln!( + std::io::stderr().lock(), + "Worker shutdown publication recovered; claims released" + )); + } + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs new file mode 100644 index 0000000000..abf03f2f04 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod open_local_workers; diff --git a/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs new file mode 100644 index 0000000000..33803a125e --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs @@ -0,0 +1,18 @@ +//! Open the installation's local process owner before accepting scheduler work. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::configuration::read_configuration; +use crate::workers::models::LocalWorkers; + +pub(crate) fn open_local_workers( + configuration_path: Option<&str>, + control_variable: String, + scheduler_enabled: bool, +) -> Result { + Ok(LocalWorkers { + configuration: read_configuration(configuration_path, scheduler_enabled)?, + active: Vec::new(), + control_variable, + }) +} diff --git a/crates/sqlbuild-cli/src/workers/mod.rs b/crates/sqlbuild-cli/src/workers/mod.rs new file mode 100644 index 0000000000..ec01b51f5c --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/mod.rs @@ -0,0 +1,5 @@ +//! Native process owners launch claim-scoped workers without transferring control-store access. + +mod _helpers; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs new file mode 100644 index 0000000000..6fb061ad4d --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -0,0 +1,47 @@ +//! Installation launch configuration and the physical children retained until reaping. + +use std::collections::BTreeMap; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::path::PathBuf; +use std::process::ExitStatus; +use std::time::Instant; + +use sqlbuild_orchestration::runs::models::RunClaim; +use tokio::process::Child; +use uuid::Uuid; + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LocalLaunchers { + pub(super) local: Vec, + #[serde(default)] + pub(super) concurrency_limits: BTreeMap, +} + +/// No executable, project path or API address is supplied by an untrusted run request. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LocalLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, + pub(super) project_dir: PathBuf, + pub(super) executable: PathBuf, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, +} + +#[derive(Debug)] +pub(super) struct LocalWorker { + pub(super) claim: RunClaim, + pub(super) child: Option, + pub(super) stopping: Option, + pub(super) exit: Option, +} + +/// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. +#[derive(Debug)] +pub(crate) struct LocalWorkers { + pub(super) configuration: LocalLaunchers, + pub(super) active: Vec, + pub(super) control_variable: String, +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 2948290095..4340e30294 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -98,10 +98,15 @@ pub(crate) async fn start_claimed_worker( .await .expect("acquire lease") .expect("leader"); - let claim = claim_next_run(&store, &lease, &BTreeMap::new()) - .await - .expect("claim") - .expect("queued work"); + let claim = claim_next_run( + &store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("queued work"); assert_eq!(claim.request.priority, -3); assert_eq!( claim.request.settings.get("full_refresh"), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs index c7f6dd2e06..70c607a995 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -7,9 +7,11 @@ use axum::response::Response; use uuid::Uuid; use crate::api::errors::ApiError; +use crate::api::models::ReadResource; use crate::credentials::main::authenticate_token::authenticate_token; use crate::credentials::models::{CredentialScope, TokenIdentity}; -use crate::store::models::ControlStore; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub(crate) async fn authenticate( State(store): State, @@ -51,3 +53,40 @@ pub(crate) fn require_location( } Ok(()) } + +/// A guessed resource ID never widens the already authenticated location and target. +pub(crate) async fn require_resource( + store: &ControlStore, + (location_id, target): (Uuid, &str), + (resource, id): (ReadResource, Uuid), +) -> Result<(), ApiError> { + let (statement, missing) = match resource { + ReadResource::Run => ( + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", + "run not found", + ), + ReadResource::Trigger => ( + "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", + "trigger not found", + ), + }; + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + statement, + vec![ + Parameter::Id(id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: missing.into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs index c8c002e3d0..06ee0574ab 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs @@ -5,17 +5,20 @@ use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::{require_location, require_resource}; use crate::api::errors::ApiError; -use crate::api::models::ReadQuery; +use crate::api::models::{ReadQuery, ReadResource, RunRecallPath, RunRecallQuery}; use crate::credentials::models::TokenIdentity; use crate::runs::main::list_run_nodes::list_run_nodes; use crate::runs::main::list_runs::list_runs; use crate::runs::models::{RunNodeRecord, RunRecord}; use crate::store::main::begin::begin; use crate::store::main::commit::commit; +use crate::store::main::list_recall_invocations::list_recall_invocations; use crate::store::main::query::query; +use crate::store::main::read_recall_view::read_recall_view; use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; #[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", params(("location_id" = Uuid, Path), ("target" = String, Path), @@ -45,23 +48,85 @@ pub(crate) async fn nodes( Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - let transaction = begin(&store, TransactionMode::Read).await?; - let (transaction, rows) = query( - transaction, - "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", - vec![ - Parameter::Id(run_id), - Parameter::Id(location_id), - Parameter::Text(target), - ], - ) - .await?; - commit(transaction).await?; - if rows.is_empty() { + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + Ok(Json(list_run_nodes(&store, run_id).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum invocation namespaces; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn invocations( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + Ok(Json( + list_recall_invocations(&store, &namespaces, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("view" = String, Path), ("invocation" = String, Query, description = "Exact scheduler or worker invocation ID"), + ("limit" = Option, Query, description = "Maximum rows; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 400, description = "Unknown Recall view"), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 404, description = "Run or invocation not found")), security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(RunRecallPath { + location_id, + target, + run_id, + view, + }): Path, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + if !namespaces.iter().any(|namespace| { + options.invocation == *namespace || options.invocation.starts_with(&format!("{namespace}/")) + }) { return Err(ApiError { status: StatusCode::NOT_FOUND, - message: "run not found".into(), + message: "run invocation not found".into(), }); } - Ok(Json(list_run_nodes(&store, run_id).await?)) + Ok(Json( + read_recall_view( + &store, + (&view, &options.invocation), + options.limit.unwrap_or(100), + ) + .await?, + )) +} + +/// Immutable claim evidence remains available after terminal publication removes liveness. +async fn run_namespaces(store: &ControlStore, run_id: Uuid) -> Result, ApiError> { + let scheduler = format!("scheduler/{run_id}"); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, claims) = query( + transaction, + "SELECT payload FROM recall_events WHERE invocation_id=$1 AND event_type='run_claimed' ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(scheduler.clone())], + ).await?; + commit(transaction).await?; + let mut namespaces = vec![scheduler]; + if let Some(claim_id) = claims + .first() + .and_then(|row| row["payload"]["claim"]["claim_id"].as_str()) + { + namespaces.push(format!("worker/{claim_id}")); + } + Ok(namespaces) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs index ec5dfa88eb..bcc5fd5bdd 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs @@ -1,23 +1,19 @@ //! Trigger definitions and immutable history are visible only within the authenticated location. use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::{require_location, require_resource}; use crate::api::errors::ApiError; -use crate::api::models::ReadQuery; +use crate::api::models::{ReadQuery, ReadResource}; use crate::credentials::models::TokenIdentity; use crate::declarations::main::list_trigger_events::list_trigger_events; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{TriggerChangeRecord, TriggerRecord}; use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::models::TickRecord; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::ControlStore; #[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", params(("location_id" = Uuid, Path), ("target" = String, Path)), @@ -45,7 +41,12 @@ pub(crate) async fn events( Query(options): Query, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - require_trigger(&store, (location_id, &target, trigger_id)).await?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; Ok(Json( list_trigger_events(&store, trigger_id, options.limit.unwrap_or(100)).await?, )) @@ -64,33 +65,13 @@ pub(crate) async fn ticks( Query(options): Query, ) -> Result>, ApiError> { require_location(&identity, location_id)?; - require_trigger(&store, (location_id, &target, trigger_id)).await?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; Ok(Json( list_ticks(&store, trigger_id, options.limit.unwrap_or(100)).await?, )) } - -async fn require_trigger( - store: &ControlStore, - (location_id, target, trigger_id): (Uuid, &str, Uuid), -) -> Result<(), ApiError> { - let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query( - transaction, - "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", - vec![ - Parameter::Id(trigger_id), - Parameter::Id(location_id), - Parameter::Text(target.into()), - ], - ) - .await?; - commit(transaction).await?; - if rows.is_empty() { - return Err(ApiError { - status: StatusCode::NOT_FOUND, - message: "trigger not found".into(), - }); - } - Ok(()) -} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index f2ce27445f..a08c8a3130 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -45,6 +45,14 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", get(run_reads::nodes), ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + get(run_reads::invocations), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + get(run_reads::recall), + ) .route( "/api/locations/{location_id}/targets/{target}/triggers", get(trigger_reads::current), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index d5070b7bb9..b39ad9fd6c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -22,6 +22,8 @@ use crate::api::_helpers::{ runs::recall, run_reads::recent, run_reads::nodes, + run_reads::invocations, + run_reads::recall, trigger_reads::current, trigger_reads::events, trigger_reads::ticks @@ -34,6 +36,27 @@ pub(crate) struct ReadQuery { pub(crate) limit: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RunRecallQuery { + pub(crate) invocation: String, + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RunRecallPath { + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) run_id: uuid::Uuid, + pub(crate) view: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum ReadResource { + Run, + Trigger, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct MutationResponse { pub applied: bool, diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs index 3a86ee6d75..9bf328e7ed 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -24,6 +24,7 @@ pub async fn claim_next_run( store: &ControlStore, lease: &SchedulerLease, limits: &BTreeMap, + scope: Option<(Uuid, &str)>, ) -> Result, ControlError> { let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { @@ -37,11 +38,14 @@ pub async fn claim_next_run( (transaction, rows) = query(transaction, &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.cancel_requested_at IS NULL \ AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ + AND ($4 IS NULL OR (r.location_id=$4 AND r.target=$5)) \ AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ ORDER BY r.priority DESC, r.queued_at, r.run_id LIMIT 100{locking}"), vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), - Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id))]).await?; + Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id)), + Parameter::OptionalId(scope.map(|(location, _)| location)), + Parameter::OptionalText(scope.map(|(_, target)| target.to_owned()))]).await?; if rows.is_empty() { commit(check_lease(transaction, lease).await?).await?; return Ok(None); diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs index 1354b035fb..101f419a95 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -15,7 +15,7 @@ pub(crate) async fn job_write_set( ) -> Result<(ControlTransaction, BTreeSet), ControlError> { let (transaction, rows) = query(transaction, "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ - WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','function')", + WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','udf','table_fn')", vec![Parameter::Id(snapshot_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(job_key.into())]).await?; Ok(( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 97d86e1251..a7989d799a 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod job_write_set; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; +pub mod observe_run; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; diff --git a/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs new file mode 100644 index 0000000000..f18e11f7a6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs @@ -0,0 +1,24 @@ +//! Observe one run and its current claim using the same database-clock snapshot. + +use uuid::Uuid; + +use crate::runs::models::RunObservation; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn observe_run( + store: &ControlStore, + run_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, "SELECT r.status, r.cancel_requested_at, l.claimed_at, l.last_heartbeat_at, $2 AS observed_at FROM sqb_runs r LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", vec![Parameter::Id(run_id), Parameter::Time(now)]).await?; + let state = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("launched run is missing".into()))?, + )?; + commit(transaction).await?; + Ok(state) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 70a560ab44..34444c32e8 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -307,3 +307,13 @@ pub(crate) struct ClaimMetadata { pub(crate) claimed_at: DateTime, pub(crate) sqb_version: String, } + +/// A physical launcher uses the database clock and current liveness, never an old run event. +#[derive(Debug, Deserialize)] +pub struct RunObservation { + pub status: RunStatus, + pub cancel_requested_at: Option>, + pub claimed_at: Option>, + pub last_heartbeat_at: Option>, + pub observed_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index 29c8180b65..b4fd01d5e5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -4,6 +4,7 @@ use tokio::sync::watch; use crate::scheduler::main::scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; +use crate::scheduler::types::RunLauncher; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -11,17 +12,25 @@ use crate::store::models::ControlStore; pub async fn run_scheduler( store: &ControlStore, mut state: SchedulerState, - mut shutdown: watch::Receiver, - mut changes: watch::Receiver<()>, + (mut shutdown, mut changes): (watch::Receiver, watch::Receiver<()>), + mut launcher: impl RunLauncher, ) -> Result<(), ControlError> { - while !*shutdown.borrow() { - let (next_state, cycle) = scheduler_cycle(store, state).await?; - state = next_state; - tokio::select! { - _ = tokio::time::sleep(cycle.wait_for) => {}, - result = changes.changed() => { if result.is_err() { break; } }, - result = shutdown.changed() => { if result.is_err() { break; } }, + let mut last_lease = None; + let result = async { + while !*shutdown.borrow() { + let (next_state, cycle) = scheduler_cycle(store, state).await?; + state = next_state; + last_lease = state.lease.clone(); + launcher.poll(store, state.lease.as_ref()).await?; + tokio::select! { + _ = tokio::time::sleep(cycle.wait_for.min(std::time::Duration::from_secs(1))) => {}, + result = changes.changed() => { if result.is_err() { break; } }, + result = shutdown.changed() => { if result.is_err() { break; } }, + } } + Ok(()) } - Ok(()) + .await; + let stopped = launcher.shutdown(store, last_lease.as_ref()).await; + result.and(stopped) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs index 3978df0a4a..cf840ff555 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; -pub(crate) mod types; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs index 274b453740..53e9334d97 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -2,5 +2,24 @@ use chrono::{DateTime, Utc}; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Physical process ownership survives leadership loss, but durable writes require a live fence. +pub trait RunLauncher { + fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; + + fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; +} + /// Expanded wall-clock times and the first omitted time, if the batch was bounded. pub(crate) type ScheduleExpansion = (Vec>, Option>); diff --git a/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs new file mode 100644 index 0000000000..ced2c62b2c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs @@ -0,0 +1,62 @@ +//! Discover invocation namespaces from retained Recall facts, including output-only attempts. + +use sqlbuild_recall::schema::constants::INVOCATION_TABLES; +use sqlbuild_recall::schema::main::table_spec::table_spec; + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// The caller authorizes exact namespace roots; prefix matching escapes LIKE patterns and includes the slash boundary. +pub async fn list_recall_invocations( + store: &ControlStore, + namespaces: &[String], + limit: u32, +) -> Result, ControlError> { + if namespaces.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + let mut parameters: Vec = Vec::new(); + let predicates = namespaces + .iter() + .map(|namespace| { + let exact = parameters.len() + 1; + parameters.push(Parameter::Text(namespace.clone())); + let pattern = namespace + .replace('!', "!!") + .replace('%', "!%") + .replace('_', "!_"); + parameters.push(Parameter::Text(format!("{pattern}/%"))); + format!( + "invocation_id=${exact} OR invocation_id LIKE ${} ESCAPE '!'", + exact + 1 + ) + }) + .collect::>() + .join(" OR "); + let tables = INVOCATION_TABLES + .iter() + .map(|table| { + format!( + "SELECT invocation_id, invocation_started_at FROM {} WHERE {predicates}", + table_spec(*table).name, + ) + }) + .collect::>() + .join(" UNION ALL "); + parameters.push(Parameter::Integer(i64::from(limit.min(1000)))); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at \ + FROM ({tables}) AS records GROUP BY invocation_id ORDER BY invocation_id LIMIT ${}", + parameters.len() + ), + parameters, + ) + .await?; + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index de522badbc..715e39afaf 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; pub(crate) mod decode_row; +pub mod list_recall_invocations; pub mod migrate_store; pub mod open_store; pub mod publish_recall; diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 65ad349b97..9b4b904f29 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -183,7 +183,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie .await .expect("elect") .expect("leader"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("work"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index 6849aae676..598dbd7496 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -13,16 +13,24 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; use sqlbuild_orchestration::declarations::models::{TriggerChange, TriggerChangeRequest}; +use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::request_run::request_run; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::publish_recall::publish_recall; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputStream, RowEnvelope}; use uuid::Uuid; use crate::helpers::{read_json, start_api, stop_api}; use crate::location_helpers::create_location; +use crate::output_helpers::chunk; +use crate::recall_helpers::recall_contract; use crate::run_helpers::{request, snapshot}; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; @@ -145,10 +153,15 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso ) .await .expect("principal token"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) - .await - .expect("claim") - .expect("work"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("work"); let worker = issue_token( store, &audit, @@ -174,13 +187,123 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso let trigger_path = format!("{prefix}/triggers"); let event_path = format!("{trigger_path}/{trigger_id}/events"); let tick_path = format!("{trigger_path}/{trigger_id}/ticks"); + let invocation_path = format!("{prefix}/runs/{}/invocations", claim.request.run_id); + let recall_path = format!( + "{prefix}/runs/{}/recall/events?invocation=worker/{}/execution", + claim.request.run_id, claim.claim_id + ); let paths = [ &run_path, &node_path, &trigger_path, &event_path, &tick_path, + &invocation_path, + &recall_path, ]; + let connection = connect_worker( + &api.url, + &worker.bearer, + (claim.request.run_id, claim.claim_id), + ) + .expect("worker HTTP connection"); + recall_contract(store, &connection, claim.claim_id).await; + let output = chunk(0, OutputStream::Stdout, b"orders worker output"); + connection + .publish_output(std::slice::from_ref(&output)) + .await + .expect("output-only namespace"); + let neighboring = RowEnvelope { + invocation_id: format!("worker/{}-other", claim.claim_id), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + publish_recall(store, &[output_row(&neighboring, &output)]) + .await + .expect("neighboring namespace"); + let invocations = read_json(&client, (&invocation_path, &token.bearer)).await; + let invocation_ids = invocations + .as_array() + .expect("run invocations") + .iter() + .map(|row| { + row["invocation_id"] + .as_str() + .expect("invocation ID") + .to_owned() + }) + .collect::>(); + assert_eq!( + invocation_ids, + vec![ + format!("scheduler/{}", claim.request.run_id), + format!("worker/{}", claim.claim_id), + format!("worker/{}/another-invocation", claim.claim_id), + format!("worker/{}/backlog", claim.claim_id), + format!("worker/{}/execution", claim.claim_id), + format!("worker/{}/native-writer", claim.claim_id), + ] + ); + let evidence = read_json(&client, (&recall_path, &token.bearer)).await; + assert_eq!(evidence.as_array().expect("worker events").len(), 1); + assert_eq!(evidence[0]["duration_ms"], json!(12.75)); + assert_eq!(evidence[0]["row_count"], json!(9)); + assert_eq!( + read_json(&client, (&format!("{recall_path}&limit=0"), &token.bearer)).await, + json!([]) + ); + for path in [ + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("scheduler/{}", foreign_run.run_id), + ), + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("worker/{}-other/execution", claim.claim_id), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", manual_ids[0]), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", foreign_run.run_id), + ), + recall_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("Recall scope denial") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + client + .get(recall_path.replace("/recall/events?", "/recall/unknown?")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("unknown view") + .status(), + StatusCode::BAD_REQUEST + ); + let scheduler_path = format!( + "{prefix}/runs/{}/recall/events?invocation=scheduler/{}", + claim.request.run_id, claim.request.run_id + ); + let history = read_json(&client, (&scheduler_path, &token.bearer)).await; + assert!( + history + .as_array() + .expect("run history") + .iter() + .any(|row| row["event_type"] == "run_claimed") + ); let runs = read_json(&client, (&run_path, &token.bearer)).await; assert_eq!( runs.as_array().expect("runs").len(), @@ -214,7 +337,7 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso assert_eq!(ticks[0]["outcome"], "requested"); assert_eq!(ticks[0]["scheduled_for"], "2020-01-03T00:00:00Z"); assert_eq!(ticks[1]["outcome"], "covered"); - for path in [&run_path, &event_path, &tick_path] { + for path in [&run_path, &event_path, &tick_path, &invocation_path] { assert_eq!( read_json(&client, (&format!("{path}?limit=1"), &token.bearer)) .await @@ -275,9 +398,14 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso node_path.replace(&manual_ids[0].to_string(), &foreign_run.run_id.to_string()), event_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), tick_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + invocation_path.replace( + &claim.request.run_id.to_string(), + &foreign_run.run_id.to_string(), + ), node_path.replace("/development/", "/validation/"), event_path.replace("/development/", "/validation/"), tick_path.replace("/development/", "/validation/"), + invocation_path.replace("/development/", "/validation/"), ] { assert_eq!( client @@ -322,6 +450,8 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso for path in [ "/api/locations/{location_id}/targets/{target}/runs", "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", "/api/locations/{location_id}/targets/{target}/triggers", "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", @@ -345,6 +475,32 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso events ); assert_eq!(read_json(&client, (&tick_path, &token.bearer)).await, ticks); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "Stop scoped read fixture".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminalize worker and release liveness"); + assert_eq!( + read_json(&client, (&invocation_path, &token.bearer)).await, + invocations, + "invocation discovery uses retained facts, not active liveness" + ); + assert_eq!( + read_json(&client, (&recall_path, &token.bearer)).await, + evidence, + "historical Recall remains authorized after the active claim is removed" + ); revoke_token(store, &audit, token.identity.token_id) .await .expect("revoke location"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 8f4bf9e993..60d5932881 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -68,7 +68,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a .await .expect("acquire") .expect("leader"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("work"); @@ -247,7 +247,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a &request((location, publication.snapshot_id), 2, 0, &["orders"]), ) .await; - let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new()) + let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim another run") .expect("queued work"); diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 3c3c62f087..67526d5c57 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -191,7 +191,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .await .expect("elect") .expect("leader"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("work"); diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs index 7f90e2f877..ec318744e7 100644 --- a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -20,7 +20,7 @@ pub(crate) async fn start_run( input: &RunRequest, ) -> RunClaim { queue(store, input).await; - let claim = claim_next_run(store, lease, &BTreeMap::new()) + let claim = claim_next_run(store, lease, &BTreeMap::new(), None) .await .expect("claim queued run") .expect("run available"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index fdf5e34d4e..43619dfc46 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -16,7 +16,103 @@ use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::terminal_helpers::finish; -use crate::test_types::{BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase}; +use crate::test_types::{ + BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, +}; + +#[test] +fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then_only_its_location_and_target_are_reserved() + { + let test_cases = [ + ScopedClaimTestCase { + description: "SQLite local dispatch cannot reserve another location or target", + backend: Backend::Sqlite, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ScopedClaimTestCase { + description: "PostgreSQL local dispatch cannot reserve another location or target", + backend: Backend::Postgres, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let other_published = publish_snapshot(store, &snapshot(other)) + .await + .expect("other location snapshot"); + let mut secondary = snapshot(location); + secondary.target = "secondary".into(); + let secondary = publish_snapshot(store, &secondary) + .await + .expect("other target snapshot"); + let wanted = request((location, published.snapshot_id), 1, 0, &["orders"]); + let mut other_target = request((location, secondary.snapshot_id), 2, 10, &["orders"]); + other_target.target = "secondary".into(); + let other_location = request((other, other_published.snapshot_id), 3, 20, &["orders"]); + for input in [&wanted, &other_target, &other_location] { + queue(store, input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("scoped claim") + .expect("matching work"); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_selected_run), + "{}", + test_case.description + ); + assert!( + claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")) + ) + .await + .expect("scope exhausted") + .is_none() + ); + finish(store, &lease, &claim).await; + let next = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("unrestricted claim") + .expect("other work was not reserved"); + assert_eq!( + next.request.run_id, + Uuid::from_u128(test_case.expected_unscoped_run), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} #[test] fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() @@ -65,8 +161,8 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol let second = second_store(store).await; let limits = BTreeMap::new(); let (first, other) = tokio::join!( - claim_next_run(store, &lease, &limits), - claim_next_run(&second, &lease, &limits) + claim_next_run(store, &lease, &limits, None), + claim_next_run(&second, &lease, &limits, None) ); let first = first.expect("first claimant").expect("claim available"); let other = other @@ -87,14 +183,14 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol test_case.description ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("blocked run stays queued") .is_none() ); queue(store, &request(scope, 4, 9, &["customers", "orders"])).await; queue(store, &request(scope, 5, 0, &["customers"])).await; - let independent = claim_next_run(store, &lease, &limits) + let independent = claim_next_run(store, &lease, &limits, None) .await .expect("try partially conflicting candidate") .expect("independent work proceeds"); @@ -109,7 +205,7 @@ fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rol .expect("orders owner"); finish(store, &lease, orders).await; finish(store, &lease, &independent).await; - let next = claim_next_run(store, &lease, &limits) + let next = claim_next_run(store, &lease, &limits, None) .await .expect("terminal cleanup committed") .expect("next priority run claims both nodes"); @@ -193,23 +289,23 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil queue(store, &input).await; } assert!( - claim_next_run(store, &lease, &BTreeMap::new()) + claim_next_run(store, &lease, &BTreeMap::new(), None) .await .is_err(), "unknown groups cannot silently become unlimited" ); - let first = claim_next_run(store, &lease, &limits) + let first = claim_next_run(store, &lease, &limits, None) .await .expect("first slot") .expect("available"); - let second = claim_next_run(store, &lease, &limits) + let second = claim_next_run(store, &lease, &limits, None) .await .expect("second slot") .expect("available"); assert_eq!(first.request.run_id, Uuid::from_u128(1)); assert_eq!(second.request.run_id, Uuid::from_u128(2)); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("group full") .is_none() @@ -220,13 +316,13 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil NonZeroU32::new(1).expect("nonzero lowered limit"), ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("high slot counts after limit reduction") .is_none() ); finish(store, &lease, &second).await; - let third = claim_next_run(store, &lease, &limits) + let third = claim_next_run(store, &lease, &limits, None) .await .expect("released group") .expect("third queued run"); @@ -276,7 +372,7 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in .expect("leader"); let limits = BTreeMap::new(); queue(store, &request(scope, 1, 0, &["orders"])).await; - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("claim blocker") .expect("available"); @@ -292,7 +388,7 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in } queue(store, &request(scope, 105, -1, &["customers"])).await; let started = std::time::Instant::now(); - let independent = claim_next_run(store, &lease, &limits) + let independent = claim_next_run(store, &lease, &limits, None) .await .expect("scan past a full blocked page") .expect("independent work"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index b38db00540..3149b775d9 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -63,7 +63,7 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks evaluate_trigger(store, &lease, trigger) .await .expect("first tick"); - let claim = claim_next_run(store, &lease, &BTreeMap::new()) + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim first run") .expect("scheduled run"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index d32ffd0472..00d20c38c3 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -88,7 +88,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene }; install_fault(store, &point).await; assert!( - claim_next_run(store, &lease, &limits).await.is_err(), + claim_next_run(store, &lease, &limits, None).await.is_err(), "{} at {table}", point.description ); @@ -117,14 +117,14 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene install_fault(store, &expiry).await; assert!( matches!( - claim_next_run(store, &lease, &limits).await, + claim_next_run(store, &lease, &limits, None).await, Err(ControlError::Fenced) ), "{}", expiry.description ); remove_fault(store, &expiry).await; - let claim = claim_next_run(store, &lease, &limits) + let claim = claim_next_run(store, &lease, &limits, None) .await .expect("retry complete claim") .expect("no leaked claims or slots"); @@ -136,7 +136,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene test_case.description ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("already claimed run is not relaunched") .is_none() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 8dca64c004..5c23383e12 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -58,7 +58,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ queue(store, &request(scope, 1, 2, &["customers", "orders"])).await; queue(store, &request(scope, 2, 1, &["customers", "orders"])).await; let limits = BTreeMap::new(); - let claim = claim_next_run(store, &lease, &limits) + let claim = claim_next_run(store, &lease, &limits, None) .await .expect("claim") .expect("available"); @@ -246,7 +246,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ before ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("claims retained") .is_none() @@ -292,7 +292,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .expect("heartbeat reports pending cancellation") ); assert!( - claim_next_run(store, &lease, &limits) + claim_next_run(store, &lease, &limits, None) .await .expect("cancel request does not release an executing run's claims") .is_none() @@ -345,7 +345,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ assert_eq!(nodes[0].status, "success"); assert_eq!(nodes[0].rows_affected, Some(17)); assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); - let following = claim_next_run(store, &lease, &limits) + let following = claim_next_run(store, &lease, &limits, None) .await .expect("claim following run") .expect("released capacity"); @@ -472,7 +472,7 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l .expect("elect") .expect("leader"); assert!( - claim_next_run(store, &lease, &BTreeMap::new()) + claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("cancelled work cannot launch") .is_none() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index e567d9d25b..228dd090a2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -80,7 +80,7 @@ fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_ start_run(store, &lease, &request(scope, 2, 0, &["customers"])).await; age_heartbeat(store, Uuid::from_u128(2), 40).await; queue(store, &request(scope, 3, 0, &["inventory"])).await; - let launching = claim_next_run(store, &lease, &BTreeMap::new()) + let launching = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("claim") .expect("launching"); @@ -247,7 +247,7 @@ fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_ .expect("takeover") .expect("successor"); assert!( - claim_next_run(store, &successor, &BTreeMap::new()) + claim_next_run(store, &successor, &BTreeMap::new(), None) .await .expect("write claims remain held after heartbeat loss and takeover") .is_none() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 0cf4c4b234..226cb57a5b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -162,7 +162,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an "orders-load".into(), std::num::NonZeroU32::new(1).expect("positive group limit"), )]); - let claim = claim_next_run(store, &lease, &limits) + let claim = claim_next_run(store, &lease, &limits, None) .await .expect("claim") .expect("queued request"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index f0112b2903..0a21b7f2ef 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -5,6 +5,13 @@ use sqlbuild_orchestration::runs::models::RunStatus; pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; +pub(crate) struct ScopedClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_selected_run: u128, + pub(crate) expected_unscoped_run: u128, +} + pub(crate) struct PriorityClaimsTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index fbe1a63283..e62bf51595 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -15,7 +15,7 @@ use crate::replica_helpers::second_store; use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store_test_types::Backend; use crate::ticks::helpers::snapshot; -use crate::ticks::test_types::TickTransactionTestCase; +use crate::ticks::test_types::{IdleLauncher, TickTransactionTestCase}; #[test] fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() @@ -85,7 +85,12 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain assert_eq!(runs.len(), test_case.expected_runs); let (stop, shutdown) = tokio::sync::watch::channel(false); let (_changed, changes) = tokio::sync::watch::channel(()); - let mut running = Box::pin(run_scheduler(&second, second_state, shutdown, changes)); + let mut running = Box::pin(run_scheduler( + &second, + second_state, + (shutdown, changes), + IdleLauncher, + )); tokio::select! { result = &mut running => panic!("loop ended before shutdown: {result:?}"), _ = tokio::time::sleep(Duration::from_millis(50)) => {}, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 95953810a5..5f10deea0d 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -15,3 +15,23 @@ pub(crate) struct TriggerControlsTestCase { pub(crate) backend: Backend, pub(crate) expected_events: usize, } + +pub(crate) struct IdleLauncher; + +impl sqlbuild_orchestration::scheduler::types::RunLauncher for IdleLauncher { + async fn poll( + &mut self, + _store: &sqlbuild_orchestration::store::models::ControlStore, + _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, + ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &sqlbuild_orchestration::store::models::ControlStore, + _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, + ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + Ok(()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs index 5f2f304c6b..0a444023a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs @@ -104,3 +104,35 @@ pub(crate) fn config_value_to_python(py: Python<'_>, value: ConfigValue) -> PyRe } } } + +/// JSON-backed configuration keeps integer widths and recursively constructs native Python values. +pub(crate) fn json_value_to_python<'py>( + py: Python<'py>, + value: &serde_json::Value, +) -> PyResult> { + use serde_json::Value; + match value { + Value::Null => Ok(py.None().into_bound(py)), + Value::Bool(value) => value.into_bound_py_any(py), + Value::String(value) => value.into_bound_py_any(py), + Value::Number(value) => match (value.as_i64(), value.as_u64()) { + (Some(value), _) => value.into_bound_py_any(py), + (None, Some(value)) => value.into_bound_py_any(py), + (None, None) => value.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py), + }, + Value::Array(items) => { + let values = items + .iter() + .map(|item| json_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, values)?.into_any()) + } + Value::Object(entries) => { + let table = PyDict::new(py); + for (key, item) in entries { + table.set_item(key, json_value_to_python(py, item)?)?; + } + Ok(table.into_any()) + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8dbd45566b..3f45271352 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -1,5 +1,6 @@ //! `sqb build`: resolve, compile, gate, execute and report one direct build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -38,6 +40,14 @@ fn run_with_providers(context: &BuildContext<'_>) -> PyResult { } fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + if let Some(worker) = &context.worker { + worker.check_cancelled(context.py())?; + if !context.get("effective_defer_clone_from")?.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + "worker defer-clone requires claimed boundary writers and is not yet supported", + )); + } + } run_defer_clone_prephase(context)?; let cache_scope: Bound<'_, PyAny> = call_with( &host::entry(context.py(), AdapterHost::RelationMetadataCacheScope)?, @@ -49,6 +59,9 @@ fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) - /// Plan and run the build, sharing one relation metadata cache from planning to the run's end. fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + if let Some(worker) = &context.worker { + worker.enforce_plan_writers(&pipeline)?; + } write_plan_text(context, &pipeline)?; enforce_build_plan_policies(context, &pipeline)?; write_compile_target(context, &pipeline)?; @@ -74,9 +87,11 @@ fn run_within_limits( request: &Bound<'_, PyAny>, tracker: &Bound<'_, PyAny>, started: Instant, + worker: Option>, ) -> PyResult { let py: Python<'_> = request.py(); - let context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + let mut context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + context.worker = worker; let limits: Bound<'_, PyAny> = context.get("execution_limits")?; let deadline: Bound<'_, PyAny> = call_with( &host::entry(py, ExecutorHost::ExecutionDeadlineScope)?, @@ -97,6 +112,15 @@ fn run_within_limits( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + run_build(py, request, None) +} + +/// A claim owner survives planning, execution and connection cleanup before terminal reporting. +pub(crate) fn run_build( + py: Python<'_>, + request: &Bound<'_, PyAny>, + worker: Option>, +) -> PyResult { let tracker: Bound<'_, PyAny> = BuildPhaseTimingTracker::new(py, None) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; @@ -107,7 +131,7 @@ pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P within(&reporting, |_| { within(&tracker.call_method0("scope")?, |_| { let started: Instant = Instant::now(); - run_within_limits(request, &tracker, started).inspect_err(|_| { + run_within_limits(request, &tracker, started, worker.clone()).inspect_err(|_| { let verbose: bool = native_field(request, "verbose") .and_then(|value| value.is_truthy()) .unwrap_or(false) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index f0a0f55d12..41e53c84da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -225,6 +225,10 @@ pub(crate) fn prepare_build_execution<'py>( ], )?), ) + .map(|mut value| { + value.worker = context.worker.clone(); + value + }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; let concurrency = context.concurrency()?.limit; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 1f1b394cb0..a2a5d45df6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -1,5 +1,6 @@ //! Resolve discovery, target, adapter, limits, connection and reporters for one build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -19,6 +20,7 @@ use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; @@ -38,6 +40,7 @@ pub(crate) struct BuildContext<'py> { pub(crate) verbose: bool, pub(crate) started: Instant, pub(crate) tracker: Bound<'py, PyAny>, + pub(crate) worker: Option>, } impl<'py> BuildContext<'py> { @@ -241,5 +244,6 @@ pub(crate) fn resolve_build_context<'py>( verbose: native_field(request, "verbose")?.is_truthy()? || debug, started, tracker: tracker.clone(), + worker: None, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index ec6bbad153..62e6f13740 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -1,7 +1,7 @@ //! Live progress rows, completion rows and runtime diagnostics of `sqb build`. use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -31,6 +31,7 @@ use crate::bindings::_helpers::build_progress::terminal::{ consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -85,11 +86,8 @@ fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { } /// The resource name and projector identity of a completion. -fn resource_identity( - result: &Bound<'_, PyAny>, - kind: ResultKind, -) -> PyResult> { - Ok(match kind { +pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { + Ok(match result_kind(result)? { ResultKind::Model => { let name: String = text(result, "model_name")?; Some((name.clone(), format!("model:{name}"))) @@ -141,6 +139,7 @@ struct CallbackState { /// Encapsulates live build progress output state and callbacks. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct BuildProgressCallbacks { + pub(crate) worker: Option>, plan: Py, stream: Py, lines: Py, @@ -453,6 +452,7 @@ impl BuildProgressCallbacks { } let lines: Bound<'_, PyAny> = shared_lines(py)?; Ok(Self { + worker: None, plan: plan.unbind(), is_tty: tty_flag(&stdout)? && !debug, stream: stream.unbind(), @@ -484,6 +484,9 @@ impl BuildProgressCallbacks { ) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker { + worker.start(py, &name, resource_kind)?; + } if let Some(projector) = &callbacks.projector { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("resource_name", name)?; @@ -579,10 +582,15 @@ impl BuildProgressCallbacks { fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker + && let Some((_, key)) = resource_identity(node_result)? + { + worker.completed(node_result, &key)?; + } let kind: ResultKind = result_kind(node_result)?; let mut result: Bound<'_, PyAny> = node_result.clone(); if let Some(projector) = &callbacks.projector - && let Some((name, identity)) = resource_identity(node_result, kind)? + && let Some((name, identity)) = resource_identity(node_result)? { let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 1f05467222..17c4dea5e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -23,12 +23,13 @@ use crate::bindings::_helpers::init_command::command::run_init_command; use crate::bindings::_helpers::janitor::janitor_command::run_janitor_command; use crate::bindings::_helpers::lineage::lineage_command::run_lineage_command; use crate::bindings::_helpers::load::load_command::run_load_command; -use crate::bindings::_helpers::location_command::command::run_location_command; +use crate::bindings::_helpers::location_command::command::run_location; use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::command::run_recall_command; +use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::review_command::command::run_review_command; @@ -313,6 +314,7 @@ fn call_route<'py>( invocation: &Invocation<'py>, route: &Route, handler: Option>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = invocation.py(); let mut positional: Vec> = Vec::new(); @@ -340,7 +342,7 @@ fn call_route<'py>( } match handler { Some(handler) => handler.call(PyTuple::new(py, positional)?, Some(&keywords)), - None => run_implementation(route.implementation, (positional, keywords)), + None => run_implementation(route.implementation, (positional, keywords), recall), } } @@ -359,6 +361,7 @@ fn argument<'py>( fn run_implementation<'py>( name: &str, arguments: (Vec>, Bound<'py, PyDict>), + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = arguments.1.py(); let request = || -> PyResult> { @@ -388,7 +391,7 @@ fn run_implementation<'py>( "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, "run_load_command" => run_load_command(py, &request()?)?, - "run_location_command" => run_location_command(py, options())?, + "run_location_command" => run_location(py, options(), Some(recall))?, "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, @@ -419,6 +422,7 @@ fn run_implementation<'py>( fn routed_command<'py>( args: &Bound<'py, PyAny>, handlers: Option<&Bound<'py, PyAny>>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); let raw_project_dir: Bound<'py, PyAny> = native_field(args, "project_dir")?; @@ -458,7 +462,7 @@ fn routed_command<'py>( if let Some(failure) = failure { return Err(user_error(py, failure)); } - call_route(&invocation, route, handler) + call_route(&invocation, route, handler, recall) } } } @@ -467,6 +471,7 @@ fn routed_command<'py>( pub(crate) fn dispatch_command<'py>( args: &Bound<'py, PyAny>, handlers: Option>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); let warehouse: Bound<'py, PyAny> = native_field(args, "warehouse")?; @@ -495,7 +500,7 @@ pub(crate) fn dispatch_command<'py>( )?, )? .into_any(); - within(&scope, |_| routed_command(args, handlers.as_ref())) + within(&scope, |_| routed_command(args, handlers.as_ref(), recall)) } /// The parsed values that select a route, read from the namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 8abf67a87f..4b4c372cee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -17,6 +17,7 @@ use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPolicy}; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; +use sqlbuild_orchestration::runs::models::RunOutcome; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; @@ -545,7 +546,11 @@ fn observed_dispatch<'py>( if let Some(session) = &invocation.recall.session { session.publish_context(false); } - return dispatch_command(&invocation.args, invocation.handlers.clone()); + return dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ); } let command: String = invocation.command_name(); drop(take_failure_cause()); @@ -601,11 +606,14 @@ fn observed_dispatch<'py>( "invocation_started", started_payload()?, )?)?; - let dispatched: PyResult> = - match dispatch_command(&invocation.args, invocation.handlers.clone()) { - Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), - dispatched => dispatched, - }; + let dispatched: PyResult> = match dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ) { + Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + dispatched => dispatched, + }; match dispatched { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); @@ -864,6 +872,30 @@ fn execute_parsed_invocation<'py>( Ok(_) | Err(_) => None, }; let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); + let worker = recall + .worker + .lock() + .map_err(|_| { + pyo3::exceptions::PyRuntimeError::new_err("worker completion lock is unavailable") + })? + .take(); + if let Some(worker) = worker { + let code = match &finished { + Ok(()) => exit_status.unwrap_or(1), + Err(_) => 1, + }; + let outcome = worker.finish(py, i32::try_from(code).unwrap_or(1))?; + host::write_console( + py, + "stderr", + &format!("Worker execution completed: {}\n", outcome.as_str()), + )?; + if !matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + dispatched?; + finished?; + return exit(py, code.max(1)); + } + } let exit_code: Bound<'py, PyAny> = dispatched?; finished?; Ok(exit_code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs new file mode 100644 index 0000000000..a472d4d696 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -0,0 +1,5 @@ +//! Frozen execution protocol values and bounded acknowledgement retries. + +pub(crate) const SKIP_ACTION: &str = "skip"; +pub(crate) const NO_CHANGE_REASON: &str = "no_change"; +pub(crate) const REPORT_ATTEMPTS: usize = 3; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 7ae7f4d521..2101d9ab7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod attachment_inputs; +pub(crate) mod constants; pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; @@ -6,6 +7,7 @@ pub(crate) mod schema_facts; pub(crate) mod snapshot; pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; pub(crate) mod snapshot_models; pub(crate) mod snapshot_resources; pub(crate) mod snapshot_worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs new file mode 100644 index 0000000000..57d0f43e90 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -0,0 +1,247 @@ +//! Frozen selection, executor callbacks and completion belong to one verified worker build. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; +use sqlbuild_core::models::ResourceKind; +use sqlbuild_orchestration::runs::models::{NodeOutcome, RunChange, RunExecution, RunOutcome}; + +use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::records::fields::field; + +/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. +#[derive(Debug)] +pub(crate) struct WorkerBuild { + pub(crate) worker: Py, + execution: RunExecution, + completed: Mutex>, +} + +impl WorkerBuild { + pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { + Ok(Arc::new(Self { + execution: worker.get().execution(worker.py())?, + worker: worker.clone().unbind(), + completed: Mutex::default(), + })) + } + + pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + if self.worker.bind(py).get().cancel_requested(py) { + return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); + } + Ok(()) + } + + /// Reject an implicitly expanded write set before any executor connection or schema change. + pub(crate) fn enforce_plan_writers( + &self, + pipeline: &Bound<'_, PyAny>, + ) -> Result<(), pyo3::PyErr> { + self.check_cancelled(pipeline.py())?; + let plan = field(pipeline, "plan_output")?; + for collection in [ + "model_entries", + "seed_entries", + "function_entries", + "source_load_entries", + ] { + for entry in field(&plan, collection)?.try_iter()? { + let entry = entry?; + let identity = field(&entry, "key")?; + let kind = field(&identity, "resource_type")? + .str()? + .to_str()? + .to_owned(); + let key = format!("{kind}:{}", field(&identity, "name")?.str()?); + if !self.execution.claim.request.write_set.contains(&key) { + return Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))); + } + if kind == ResourceKind::Model.as_str() + && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION + { + let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(pipeline.py(), &key, (outcome, None, None))?; + } + } + } + Ok(()) + } + + fn complete( + &self, + py: Python<'_>, + key: &str, + (outcome, rows_affected, error_code): (NodeOutcome, Option, Option), + ) -> PyResult<()> { + if !self.execution.claim.request.selection.contains(key) { + return Ok(()); + } + if self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("node-finished/{key}"), + RunChange::NodeFinished { + node_key: key.into(), + outcome, + rows_affected, + error_code, + skipped_because_of: None, + }, + )?; + self.completed + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), outcome); + Ok(()) + } + + pub(crate) fn start( + &self, + py: Python<'_>, + name: &str, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.check_cancelled(py)?; + let kind = field(resource_kind, "value")?.extract::()?; + let kind = match kind.as_str() { + "table" | "view" | "snapshot" | "custom" => "model", + kind => kind, + }; + let key = format!("{kind}:{name}"); + if self.execution.claim.request.selection.contains(&key) { + self.worker.bind(py).get().change( + py, + &format!("node-started/{key}"), + RunChange::NodeStarted { node_key: key }, + )?; + } + Ok(()) + } + + /// Existing native progress callbacks publish selected-node evidence without Python relays. + pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if self.execution.claim.request.selection.contains(key) { + let status: String = field(result, "status")?.str()?.extract()?; + let outcome = match status.as_str() { + "success" => { + if result.hasattr("warning_messages")? + && field(result, "warning_messages")?.is_truthy()? + { + NodeOutcome::Warn + } else { + NodeOutcome::Success + } + } + "failed" => { + if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + } + } + "skipped" => NodeOutcome::Skipped, + _ => { + return Err(PyValueError::new_err(format!( + "unsupported worker node outcome {status}" + ))); + } + }; + let rows = match result.getattr_opt("rows_affected")? { + Some(value) => value.extract()?, + None => None, + }; + let error = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete(result.py(), key, (outcome, rows, error))?; + } + Ok(()) + } + + /// Never call missing selected nodes successful merely because the executor returned zero. + pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + for key in &self.execution.claim.request.selection { + self.complete( + py, + key, + ( + if cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + None, + None, + ), + )?; + } + let outcome = { + let completed = self.completed.lock().map_err(|_| poisoned())?; + let good = completed + .values() + .any(|value| matches!(value, NodeOutcome::Success | NodeOutcome::Warn)); + let bad = completed.values().any(|value| { + matches!( + value, + NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled + ) + }); + if cancelled { + RunOutcome::Cancelled + } else if bad && good { + RunOutcome::Partial + } else if bad || exit_code != 0 { + RunOutcome::Failed + } else if completed.values().any(|value| *value == NodeOutcome::Warn) { + RunOutcome::Warn + } else { + RunOutcome::Success + } + }; + Ok(outcome) + } + + /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. + pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let outcome = self.outcome(py, exit_code)?; + self.worker.bind(py).get().change( + py, + "worker-finished", + RunChange::Finished { + outcome, + reason: "worker executor completed and closed its connections".into(), + exit_code: Some(i64::from(exit_code)), + }, + )?; + Ok(outcome) + } +} + +fn poisoned() -> pyo3::PyErr { + PyRuntimeError::new_err("worker build outcome lock is unavailable") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs index 966c3709e2..5c5da15456 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; use uuid::Uuid; +use crate::bindings::_helpers::compiled_project::constants::REPORT_ATTEMPTS; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; struct WorkerOwner { @@ -26,6 +27,61 @@ pub(crate) struct NativeOrchestrationWorker { } impl NativeOrchestrationWorker { + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub(crate) fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub(crate) fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + Ok(owner + .as_ref() + .ok_or_else(unavailable)? + .session + .execution + .clone()) + }) + } + + /// A bounded retry keeps the exact event identity across an uncertain acknowledgement. + pub(crate) fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(sqlbuild_orchestration::worker::errors::WorkerError::Transport(_)) + if attempts < REPORT_ATTEMPTS => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. pub(crate) fn start( snapshot: &NativeOrchestrationSnapshot, @@ -77,7 +133,7 @@ impl NativeOrchestrationWorker { } /// Cancellation remains sticky; a closed worker cannot authorize further work. - fn cancel_requested(&self, py: Python<'_>) -> bool { + pub(crate) fn cancel_requested(&self, py: Python<'_>) -> bool { py.detach(|| { self.owner.lock().map_or(true, |owner| { owner diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index e5673f87ea..fdd7526b6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -4,7 +4,7 @@ use std::path::Path; use std::process::{Command, Stdio}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; @@ -18,10 +18,12 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; use crate::bindings::_helpers::location_command::runtime::run_runtime; +use crate::bindings::_helpers::location_command::worker::execute_worker; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; +use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; @@ -41,9 +43,11 @@ fn metadata( .parse() .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; let target: String = arguments.value("selected_target")?.extract()?; + let action = arguments.value("action")?.extract::>()?; let generation = arguments .value("generation")? .extract::>()? + .or_else(|| (action.as_deref() == Some("execute")).then_some(1)) .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; if target.trim().is_empty() || generation < 1 { return Err(PyValueError::new_err( @@ -101,7 +105,7 @@ fn compile_location<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, input: &SnapshotInput, -) -> PyResult { +) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> { status(py, "Discovering location project...")?; let ProjectAdapter { project_dir, @@ -128,7 +132,7 @@ fn compile_location<'py>( None, )?; status(py, "Location snapshot compiled.")?; - Ok(snapshot) + Ok((snapshot, discovered)) } pub(super) fn location_arguments<'py>( @@ -147,7 +151,7 @@ pub(super) fn location_arguments<'py>( "generation", "server_url", ], - &["image_id", "action"], + &["image_id", "action", "run_id", "claim_id"], ), ) } @@ -177,7 +181,7 @@ pub(super) fn publish_location( .value("project_dir")? .extract::()?; let input = metadata(py, arguments, &directory)?; - let snapshot = compile_location(py, arguments, &input)?; + let (snapshot, _) = compile_location(py, arguments, &input)?; status(py, "Publishing location snapshot...")?; let record = snapshot.publication( py, @@ -201,8 +205,41 @@ pub(super) fn publish_location( pub(crate) fn run_location_command<'py>( py: Python<'py>, options: Option>, +) -> PyResult { + run_location(py, options, None) +} + +/// CLI dispatch retains the verified worker through invocation history delivery. +pub(crate) fn run_location<'py>( + py: Python<'py>, + options: Option>, + recall: Option<&InvocationRecall>, ) -> PyResult { let arguments = location_arguments(py, options.clone())?; + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("execute") + { + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, &arguments, &directory)?; + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let identity = ( + arguments.value("run_id")?.extract::()?, + arguments.value("claim_id")?.extract::()?, + ); + let (snapshot, discovered) = compile_location(py, &arguments, &input)?; + return execute_worker( + py, + (&snapshot, discovered), + &arguments, + (&bearer, identity, recall), + ); + } if arguments .value("action")? .extract::>()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index f80ce86450..63f708bfc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod command; mod constants; mod runtime; +mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs new file mode 100644 index 0000000000..472717ab03 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -0,0 +1,154 @@ +//! Execute only the API's frozen request after compiling and verifying the unmodified baseline. + +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_orchestration::runs::models::RunOutcome; + +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::build_command::command::run_build; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::native_host::host::{entry, write_console}; +use crate::bindings::_helpers::recall::session::InvocationRecall; +use crate::bindings::_helpers::records::construct::new_record; +use crate::bindings::_helpers::records::ids::RecordId; + +fn build_request<'py>( + py: Python<'py>, + discovered: Bound<'py, PyAny>, + arguments: &KeywordArguments<'py>, + worker: &Bound<'py, NativeOrchestrationWorker>, +) -> PyResult> { + let execution = worker.get().execution(py)?; + let frozen = execution.claim.request; + for name in frozen.settings.keys() { + if !matches!( + name.as_str(), + "full_refresh" + | "fail_fast" + | "concurrency" + | "cli_vars" + | "run_tests" + | "run_audits" + | "load_sources" + | "reload_sources" + | "include_python" + | "allow_snapshot_full_refresh" + | "allow_snapshot_schema_change" + | "allow_table_type_downgrade" + | "allow_retention_decrease" + | "allow_missing_migration_origin" + | "max_microbatches" + | "test_timeout" + ) { + return Err(PyValueError::new_err(format!( + "unsupported worker build setting {name}" + ))); + } + } + let settings = PyDict::new(py); + for (name, value) in &frozen.settings { + settings.set_item(name, json_value_to_python(py, value)?)?; + } + settings.set_item("project_dir", arguments.value("project_dir")?)?; + settings.set_item("selected_target", frozen.target)?; + let mut selectors: Vec = Vec::new(); + for key in &frozen.selection { + let (_, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; + selectors.push(name.to_owned()); + } + settings.set_item("select", selectors)?; + settings.set_item("discovered_inputs", discovered)?; + settings.set_item("no_color", true)?; + settings.set_item("json_output", true)?; + new_record(py, RecordId::BuildCommandRequest, (), Some(&settings)) +} + +/// The heartbeat is native; only driver cancellation crosses the executor host boundary. +fn run_cancellable( + py: Python<'_>, + worker: &Arc, + request: &Bound<'_, PyAny>, +) -> PyResult { + let state = Arc::clone(worker); + let (stop, stopping) = mpsc::channel::<()>(); + let thread = std::thread::Builder::new() + .name("sqb-worker-cancel".into()) + .spawn(move || { + while matches!( + stopping.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + let _ = Python::attach(|py| -> PyResult<()> { + if state.worker.bind(py).get().cancel_requested(py) { + call_with(&entry(py, ExecutorHost::CancelRunningStatements)?, &[])?; + } + Ok(()) + }); + } + }) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = run_build(py, request, Some(Arc::clone(worker))); + drop(stop); + py.detach(|| thread.join()) + .map_err(|_| PyRuntimeError::new_err("worker cancellation thread failed"))?; + result +} + +pub(super) fn execute_worker<'py>( + py: Python<'py>, + (snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>), + arguments: &KeywordArguments<'py>, + (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), +) -> PyResult { + write_console(py, "stderr", "Verifying frozen run inputs...\n")?; + let owner = Bound::new( + py, + NativeOrchestrationWorker::start( + snapshot, + py, + (&arguments.value("server_url")?.extract::()?, bearer), + identity, + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + )?; + let worker = WorkerBuild::new(&owner)?; + if let Some(recall) = recall { + *recall + .worker + .lock() + .map_err(|_| PyRuntimeError::new_err("worker completion lock is unavailable"))? = + Some(Arc::clone(&worker)); + } + let result = (|| { + let request = build_request(py, discovered, arguments, &owner)?; + write_console( + py, + "stderr", + "Run inputs verified. Executing frozen selection...\n", + )?; + run_cancellable(py, &worker, &request) + })(); + let code = result.as_ref().copied().unwrap_or(1); + let outcome = if recall.is_some() { + worker.outcome(py, code)? + } else { + worker.finish(py, code)? + }; + result.map(|code| { + if matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + code + } else { + code.max(1) + } + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 85714984a5..43b2fafcfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -36,6 +36,7 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::clickhouse_store::{ @@ -56,6 +57,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] pub(crate) struct InvocationRecall { + /// The verified worker remains owned until Recall has drained and its claim can be released. + pub(crate) worker: Arc>>>, pub(crate) session: Option>, pub(crate) target: Option, pub(crate) adapter: Option, @@ -409,35 +412,38 @@ fn quarantined_notice(path: &Path) -> Option { fn started_session( py: Python<'_>, (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), - (settings, project_dir): (&RecallSettings, &Path), + (settings, project_dir, hosted): (&RecallSettings, &Path, Option), ) -> PyResult>> { - let store: SessionStore = match &settings.store { - StoreLocation::Sqlite { path } => { - notify(py, quarantined_notice(path)); - if let Some(reason) = store_location_problem(path) { - notify(py, permission_notice((settings, project_dir), &reason)); + let store: SessionStore = match hosted { + Some(store) => store, + None => match &settings.store { + StoreLocation::Sqlite { path } => { + notify(py, quarantined_notice(path)); + if let Some(reason) = store_location_problem(path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } + SessionStore::Sqlite(path.clone()) + } + StoreLocation::ClickHouse { + config, database, .. + } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { + location: ClickHouseLocation::new(config, database, settings.retention_days), + retention_notice: notice_due, + project_dir: project_dir.to_path_buf(), + information: Mutex::default(), + })), + StoreLocation::Unsupported { + connection, + adapter, + } => { + notify( + py, + unsupported_notice((settings, project_dir), (connection, adapter)), + ); return Ok(None); } - SessionStore::Sqlite(path.clone()) - } - StoreLocation::ClickHouse { - config, database, .. - } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { - location: ClickHouseLocation::new(config, database, settings.retention_days), - retention_notice: notice_due, - project_dir: project_dir.to_path_buf(), - information: Mutex::default(), - })), - StoreLocation::Unsupported { - connection, - adapter, - } => { - notify( - py, - unsupported_notice((settings, project_dir), (connection, adapter)), - ); - return Ok(None); - } + }, }; let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); @@ -462,13 +468,44 @@ fn started_session( max_output_bytes: settings.max_output_bytes, heartbeat_interval: heartbeat_interval(), setup_budget: Duration::ZERO, - maintenance: true, + maintenance: !worker_invocation(args, facts)?, defer_context: true, audit_evidence: settings.audit_evidence, }); Ok(Some(Arc::new(session))) } +fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult { + Ok(facts.command.as_deref() == Some("location") + && attribute(args, "location_command")? + .extract::>()? + .as_deref() + == Some("execute")) +} + +fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { + let identity = ( + attribute(args, "location_run_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("run ID must be a UUID"))?, + attribute(args, "location_claim_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("claim ID must be a UUID"))?, + ); + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let url = attribute(args, "location_server_url")?.extract::()?; + args.py().detach(|| { + sqlbuild_orchestration::worker::main::connect_worker::connect_worker( + &url, &bearer, identity, + ) + .map(|client| client.recall_store()) + .map_err(|error| PyValueError::new_err(error.to_string())) + }) +} + /// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. pub(crate) fn start_invocation_recall( py: Python<'_>, @@ -482,7 +519,7 @@ pub(crate) fn start_invocation_recall( selected_target: text_value(&args.getattr("target")?)?, purpose: StorePurpose::Record, }; - let settings: RecallSettings = match resolve_recall_settings(&request) { + let mut settings: RecallSettings = match resolve_recall_settings(&request) { Ok(settings) => settings, Err(error) if error.invalid_configuration => { return Ok(InvocationRecall { @@ -492,14 +529,25 @@ pub(crate) fn start_invocation_recall( } Err(_unreachable_store) => return Ok(InvocationRecall::default()), }; + let hosted = if worker_invocation(args, facts)? { + settings.enabled = true; + Some(worker_store(args)?) + } else { + None + }; let session: Option> = if settings.enabled && settings.project_name.is_some() { - started_session(py, (args, identity, facts), (&settings, &project_dir))? + started_session( + py, + (args, identity, facts), + (&settings, &project_dir, hosted), + )? } else { None }; let audit_evidence: AuditEvidence = settings.audit_evidence; Ok(InvocationRecall { + worker: Arc::default(), session, target: settings.target.clone(), adapter: settings.project_adapter.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index a59d0e0da2..349e4ebf1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -12,6 +12,7 @@ use sqlbuild_rules::configuration::errors::RulesConfigLoadError; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record; @@ -64,31 +65,6 @@ fn option_scalar<'py>(py: Python<'py>, value: &Value) -> PyResult(py: Python<'py>, value: &Value) -> PyResult> { - match value { - Value::Array(items) => { - let values: Vec> = items - .iter() - .map(|item| option_item(py, item)) - .collect::>()?; - Ok(pyo3::types::PyList::new(py, values)?.into_any()) - } - Value::Object(entries) => { - let table: Bound<'py, PyDict> = PyDict::new(py); - for (key, item) in entries { - table.set_item(key, option_item(py, item)?)?; - } - Ok(table.into_any()) - } - Value::Null => Ok(py.None().into_bound(py)), - Value::Number(number) if number.as_i64().is_none() && number.as_u64().is_none() => { - number.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py) - } - other => Ok(option_scalar(py, other)?.unwrap_or_else(|| py.None().into_bound(py))), - } -} - fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { let result: Bound<'py, PyDict> = PyDict::new(py); for (code, options) in &config.rule_options { @@ -97,7 +73,7 @@ fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> = items .iter() - .map(|item| option_item(py, item)) + .map(|item| json_value_to_python(py, item)) .collect::>()?; values.set_item(name, PyTuple::new(py, tuple)?)?; } else if let Some(scalar) = option_scalar(py, value)? { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 66bcebbc27..6edc268f3f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,7 +1,9 @@ """Publication cases and isolated server credentials.""" -from dataclasses import dataclass -from typing import NamedTuple +from dataclasses import dataclass, field +from pathlib import Path +from subprocess import Popen +from typing import Literal, NamedTuple class SnapshotApi(NamedTuple): @@ -16,6 +18,14 @@ class ClaimedWorker(NamedTuple): bearer: str +class ScheduledWorker(NamedTuple): + api: SnapshotApi + project: Path + control_store: Path + process: Popen[bytes] + log: Path + + @dataclass(frozen=True) class SnapshotPublicationTestCase: description: str @@ -65,3 +75,16 @@ class LocationWorkTestCase: description: str database: str expected_connection: bool + + +@dataclass(frozen=True) +class WorkerExecutionTestCase: + description: str + model_sql: str + warehouse_query: str + expected_status: str + expected_rows: list[tuple[int]] + expected_node_count: int = 1 + files: dict[str, str] = field(default_factory=dict) + store_setup: str = "" + launcher: Literal["native", "uncooperative"] = "native" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index d10a50b37c..24c11ac8ff 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -6,12 +6,13 @@ import json import os import secrets +import signal import sqlite3 import subprocess import sys import time -from collections.abc import Iterator -from contextlib import contextmanager +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path @@ -21,10 +22,16 @@ from urllib.request import Request, urlopen from uuid import uuid4 +import duckdb +import pytest + from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, + ScheduledWorker, SnapshotApi, + WorkerExecutionTestCase, ) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files class RejectedPublicationOutput(StringIO): @@ -210,6 +217,183 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: return result +def native_launcher(directory: Path, executable: str) -> AbstractContextManager[str]: + """Use the installed native launcher without a synthetic child.""" + del directory + return nullcontext(executable) + + +@contextmanager +def uncooperative_launcher(directory: Path, executable: str) -> Iterator[str]: + """Own a synthetic process that ignores TERM, including cleanup after a failed assertion.""" + del executable + script: Path = directory / "worker" + _ = script.write_text( + f"#!{sys.executable}\nimport os, signal, time\nfrom pathlib import Path\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "Path('worker.pid').write_text(str(os.getpid()))\n" + "time.sleep(90)\n" + ) + script.chmod(0o700) + try: + yield str(script) + finally: + with suppress(FileNotFoundError, ProcessLookupError): + os.kill(int((directory / "project/worker.pid").read_text()), signal.SIGKILL) + + +@contextmanager +def scheduled_worker( + directory: Path, monkeypatch: pytest.MonkeyPatch, test_case: WorkerExecutionTestCase +) -> Iterator[ScheduledWorker]: + """Declare, publish and schedule isolated DuckDB work without provisioning claims.""" + project: Path = directory / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + launchers: dict[str, Callable[[Path, str], AbstractContextManager[str]]] = { + "native": native_launcher, + "uncooperative": uncooperative_launcher, + } + with ( + snapshot_api(directory / "control") as api, + launchers[test_case.launcher](directory, executable) as launch_executable, + ): + published: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "location", + "publish", + "--project-dir", + str(project), + "--location-id", + api.location_id, + "--target", + "development", + "--server", + api.url, + "--generation", + "1", + ], + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert published.returncode == 0, published.stderr + configuration: Path = directory / "launchers.json" + _ = configuration.write_text( + json.dumps( + { + "local": [ + { + "location_id": api.location_id, + "target": "development", + "project_dir": str(project), + "executable": launch_executable, + "server_url": api.url, + "max_workers": 1, + } + ] + } + ) + ) + control_store: Path = directory / "control/.sqlbuild/control.sqlite3" + with sqlite3.connect(control_store) as control: + _ = control.executescript(test_case.store_setup) + monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") + log: Path = directory / "scheduler.log" + with location_process( + [ + executable, + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(configuration), + ], + log, + directory / "scheduler.stdout", + ) as process: + yield ScheduledWorker(api, project, control_store, process, log) + + +def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -> dict[str, Any]: + """Run/node outcomes, Recall, claims and the physical warehouse agree.""" + api, project, control_store, process, log = worker + run: dict[str, Any] = wait_for_run(api, process, log) + assert run["node_counts_json"] == {test_case.expected_status: test_case.expected_node_count}, ( + log.read_text() + ) + assert run["job_key"] == "annual_orders" + assert "Worker launched for run" in log.read_text() + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen( + Request(f"{runs_url}/{run['run_id']}/nodes", headers=headers), timeout=5 + ) as response: + nodes: list[dict[str, Any]] = json.load(response) + assert nodes[0]["node_key"] == "model:orders" + assert nodes[0]["started_at"] is not None + assert nodes[0]["ended_at"] is not None + with sqlite3.connect(control_store) as control: + contexts: list[tuple[str | None]] = control.execute( + "SELECT external_context FROM recall_invocations WHERE command = 'location'" + ).fetchall() + assert contexts, log.read_text() + assert all( + context is not None and json.loads(context)["run_id"] == run["run_id"] + for (context,) in contexts + ) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + return run + + +def wait_for_run( + api: SnapshotApi, + process: subprocess.Popen[bytes], + log: Path, + *, + pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), +) -> dict[str, Any]: + """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 30 + current: dict[str, Any] = {"status": "queued"} + while current["status"] in pending: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + current = next(iter(json.load(response)), current) + time.sleep(0.02) + return current + + def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any]) -> ClaimedWorker: """Provision only the launch boundary; verification and lifecycle reporting use real HTTP.""" run_id: str = str(uuid4()) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index b357d3504a..9e8dad99f7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -1,6 +1,12 @@ """Retained compiler baselines cross the native worker boundary before lifecycle writes.""" import json +import os +import signal +import sqlite3 +import subprocess +import sys +import time from pathlib import Path from typing import Any from urllib.request import Request, urlopen @@ -23,10 +29,15 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, SnapshotWorkerTestCase, + WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( claimed_worker, + scheduled_worker, snapshot_api, + wait_for_log, + wait_for_run, + worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -146,3 +157,267 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ result: dict[str, Any] = json.load(response)[0] assert result["status"] == "success" assert result["node_counts_json"] == {"success": 1} + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "successful model writes actual rows", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('23')) t(value)", + "SELECT amount FROM orders", + "success", + [(23,)], + ), + WorkerExecutionTestCase( + "warehouse error retains failed node evidence", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + ), + WorkerExecutionTestCase( + "scalar function dependency is claimed and executed", + 'SELECT __udf("order_total")(23) AS amount', + "SELECT amount FROM orders", + "success", + [(46,)], + expected_node_count=2, + files={ + "functions/sql/order_total.sql": ( + "FUNCTION (description 'Order total.', " + "arguments (amount BIGINT), returns BIGINT);\nSELECT amount * 2\n" + ) + }, + ), + WorkerExecutionTestCase( + "table function dependency is claimed and executed", + 'SELECT amount FROM __table_fn("order_amounts")()', + "SELECT amount FROM orders", + "success", + [(31,)], + expected_node_count=2, + files={ + "functions/sql/order_amounts.sql": ( + "FUNCTION (description 'Order amounts.', returns table (amount BIGINT));\n" + "SELECT CAST(31 AS BIGINT) AS amount\n" + ) + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_real_warehouse_results( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) + ) + assert active["status"] == "running", worker.log.read_text() + deadline: float = time.monotonic() + 15 + with sqlite3.connect(worker.control_store) as control: + while not control.execute( + "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", + ("%range(10000000000)%",), + ).fetchone(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + cancelled: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "cancel-run", + active["run_id"], + "--event-id", + str(uuid4()), + "--reason", + "Cancel the active test statement.", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert cancelled.returncode == 0, cancelled.stderr + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "credential failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + ), + ), + WorkerExecutionTestCase( + "launch publication failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_launch BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'launched' " + "BEGIN SELECT RAISE(ABORT, 'synthetic launch publication failure'); END;" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_committed_claim_when_launch_preparation_fails_then_unstarted_work_is_terminal_and_unreserved( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert worker.process.wait(timeout=15) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + assert runs[0]["node_counts_json"] == {"skipped": 1} + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert not (worker.project / "warehouse.duckdb").exists() + assert "Worker launched for run" not in worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "failed terminal publication keeps the owner alive until recovery", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_stopped_worker_when_terminal_publication_fails_then_shutdown_retains_and_retries_its_claim( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + assert worker.process.poll() is None, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (1,) + _ = control.execute("DROP TRIGGER reject_terminal") + assert worker.process.wait(timeout=10) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX TERM/KILL escalation") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "uncooperative child is reaped after a lease-preserving hard kill", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_uncooperative_worker_when_scheduler_stops_then_the_fence_and_claims_survive_until_reaping( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pidfile: Path = worker.project / "worker.pid" + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + pid: int = int(pidfile.read_text()) + worker.process.send_signal(signal.SIGTERM) + time.sleep(32) + assert worker.process.poll() is None, worker.log.read_text() + os.kill(pid, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute( + "SELECT julianday(expires_at) > julianday('now') FROM sqb_scheduler_lease" + ).fetchone() == (1,) + assert worker.process.wait(timeout=40) == 0, worker.log.read_text() + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) From 75f8e0ab3e7c3009c7fe5447a7ee089584cabd73 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:51:38 +0000 Subject: [PATCH 035/138] feat: supervise Docker workers across scheduler restarts --- .github/workflows/verify.yml | 6 + crates/sqlbuild-cli/Cargo.toml | 2 +- .../src/native_commands/_helpers/execution.rs | 8 +- .../src/workers/_helpers/child.rs | 57 ++++ .../src/workers/_helpers/configuration.rs | 58 +++- .../src/workers/_helpers/docker.rs | 205 ++++++++++++++ .../src/workers/_helpers/invocation.rs | 47 ++++ .../sqlbuild-cli/src/workers/_helpers/mod.rs | 6 +- .../src/workers/_helpers/recovery.rs | 64 +++++ .../src/workers/_helpers/shutdown.rs | 49 +++- .../_helpers/{local.rs => supervisor.rs} | 161 ++++++----- crates/sqlbuild-cli/src/workers/constants.rs | 3 + crates/sqlbuild-cli/src/workers/main/mod.rs | 2 +- ...{open_local_workers.rs => open_workers.rs} | 11 +- crates/sqlbuild-cli/src/workers/mod.rs | 1 + crates/sqlbuild-cli/src/workers/models.rs | 68 ++++- .../tests/native_commands/worker/helpers.rs | 128 +++++++-- .../native_commands/worker/test_recovery.rs | 79 ++++++ .../native_commands/worker/test_types.rs | 7 + .../native_commands/worker/test_worker.rs | 2 + .../src/runs/main/list_run_claims.rs | 39 +++ .../src/runs/main/mod.rs | 1 + .../tests/runs/test_claims.rs | 32 +++ .../publication/_test_types.py | 26 +- .../publication/docker/__init__.py | 1 + .../publication/docker/helpers.py | 126 +++++++++ .../compiled_project/publication/helpers.py | 107 ++++++-- .../publication/test_docker_worker.py | 252 ++++++++++++++++++ .../publication/test_worker.py | 39 +-- 29 files changed, 1400 insertions(+), 187 deletions(-) create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/child.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/docker.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/invocation.rs create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/recovery.rs rename crates/sqlbuild-cli/src/workers/_helpers/{local.rs => supervisor.rs} (66%) create mode 100644 crates/sqlbuild-cli/src/workers/constants.rs rename crates/sqlbuild-cli/src/workers/main/{open_local_workers.rs => open_workers.rs} (62%) create mode 100644 crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3f51d3fbb9..73205dc120 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -283,7 +283,13 @@ jobs: run: make check-ci - name: Run native checks run: make rust-check + - name: Prepare Docker worker tests + run: | + docker info > /dev/null + docker pull alpine:3.23 - name: Run unit and integration tests + env: + SQLBUILD_REQUIRE_DOCKER_TESTS: "1" run: make test e2e_shards: diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index d5e621fd80..5568005174 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -19,6 +19,7 @@ path = "src/bin/sqlbuild.rs" blake3.workspace = true chrono.workspace = true getrandom.workspace = true +reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } sqlbuild-build-features.workspace = true @@ -40,7 +41,6 @@ ctrlc.workspace = true same-file.workspace = true [dev-dependencies] -reqwest.workspace = true rusqlite.workspace = true tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 41b1df3e3f..7de7f07935 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -48,8 +48,8 @@ use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; -use crate::workers::main::open_local_workers::open_local_workers; -use crate::workers::models::LocalWorkers; +use crate::workers::main::open_workers::open_workers; +use crate::workers::models::Workers; /// Request one publication without transferring interpreter ownership to a cancellable future. pub(crate) async fn compile_generation( @@ -314,8 +314,8 @@ pub(crate) async fn open_control_store( } /// Decode privileged CLI configuration before handing process ownership to the workers domain. -pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { - open_local_workers( +pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { + open_workers( text(namespace, "control_launchers").as_deref(), text(namespace, "control_database_env") .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs new file mode 100644 index 0000000000..a6ca868c82 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -0,0 +1,57 @@ +//! Launchers share physical exit observation and termination, not durable lifecycle authority. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::supervisor::{exit_code, terminate}; +use crate::workers::models::{WorkerChild, WorkerExit}; + +impl WorkerExit { + pub(in crate::workers) const fn code(self) -> Option { + match self { + Self::Code(code) => Some(code), + Self::Absent => None, + } + } +} + +impl WorkerChild { + pub(in crate::workers) async fn try_wait( + &mut self, + ) -> Result, ControlError> { + match self { + Self::Local(child) => Ok(child + .try_wait()? + .map(|exit| WorkerExit::Code(exit_code(exit)))), + Self::Docker(child) => child.try_wait().await, + } + } + + pub(in crate::workers) async fn wait(&mut self) -> Result { + if let Self::Local(child) = self { + return Ok(WorkerExit::Code(exit_code(child.wait().await?))); + } + loop { + if let Some(exit) = self.try_wait().await? { + return Ok(exit); + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + + pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => terminate(child), + Self::Docker(child) => child.signal("SIGTERM").await, + } + } + + pub(in crate::workers) async fn kill(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => { + child.start_kill()?; + Ok(()) + } + Self::Docker(child) => child.signal("SIGKILL").await, + } + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs index bf486eeb40..7377d5d941 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -5,36 +5,78 @@ use std::collections::BTreeSet; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; -use crate::workers::models::LocalLaunchers; +use crate::workers::models::{Launcher, LauncherScope, Launchers}; + +impl<'a> Launcher<'a> { + pub(in crate::workers) fn scope(self) -> LauncherScope<'a> { + match self { + Self::Local(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + Self::Docker(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + } + } +} + +impl Launchers { + pub(in crate::workers) fn locations(&self) -> impl Iterator> { + self.local + .iter() + .map(Launcher::Local) + .chain(self.docker.iter().map(Launcher::Docker)) + } +} pub(in crate::workers) fn read_configuration( path: Option<&str>, scheduler_enabled: bool, -) -> Result { - let configuration: LocalLaunchers = match path { +) -> Result { + let configuration: Launchers = match path { Some(path) => serde_json::from_slice(&std::fs::read(path)?) .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, - None => LocalLaunchers { + None => Launchers { local: Vec::new(), + docker: Vec::new(), concurrency_limits: Default::default(), }, }; - if !scheduler_enabled && !configuration.local.is_empty() { + if !scheduler_enabled && configuration.locations().next().is_some() { return Err(ControlError::Invalid( "launchers require the embedded scheduler".into(), )); } let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); for location in &configuration.local { - if location.target.trim().is_empty() - || !location.project_dir.is_absolute() + if !location.project_dir.is_absolute() || !location.project_dir.is_dir() || !location.executable.is_absolute() || !location.executable.is_file() - || !scopes.insert((location.location_id, location.target.as_str())) { return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); } } + for location in configuration.locations() { + let scope = location.scope(); + if scope.target.trim().is_empty() || !scopes.insert((scope.location_id, scope.target)) { + return Err(ControlError::Invalid( + "launchers require unique location/target scopes".into(), + )); + } + if let Launcher::Docker(location) = location + && (!location.socket.is_absolute() + || location.location_container.trim().is_empty() + || location.executable.trim().is_empty()) + { + return Err(ControlError::Invalid("Docker launchers require an absolute socket path, location container and executable".into())); + } + } Ok(configuration) } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs new file mode 100644 index 0000000000..7b0d5c9633 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs @@ -0,0 +1,205 @@ +//! The privileged role talks to a local Docker daemon; workers never receive its socket. + +use std::path::Path; + +use reqwest::{Method, StatusCode}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::constants::IMAGE_ID_HEX_LENGTH; +use crate::workers::models::{DockerLauncher, DockerWorker, Launcher, WorkerExit}; + +impl DockerWorker { + pub(in crate::workers) fn prepare( + location: &DockerLauncher, + claim: &RunClaim, + ) -> Result { + let image_id = claim.request.image_id.as_deref().unwrap_or_default(); + match image_id.strip_prefix("sha256:") { + Some(digest) + if digest.len() == IMAGE_ID_HEX_LENGTH + && digest.as_bytes().iter().all(u8::is_ascii_hexdigit) => {} + _ => { + return Err(ControlError::Invalid( + "Docker workers require a snapshot pinned to a full sha256 image ID".into(), + )); + } + } + Ok(Self { + client: docker_client(&location.socket)?, + name: format!( + "sqlbuild-{}-{}", + claim.request.location_id, claim.request.run_id + ), + claim_id: claim.claim_id, + image_id: image_id.into(), + }) + } + + pub(in crate::workers) async fn launch( + &self, + location: &DockerLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, + ) -> Result<(), ControlError> { + let parent: Value = self + .request( + Method::GET, + &format!("containers/{}/json", location.location_container), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + let project = parent["Config"]["WorkingDir"] + .as_str() + .filter(|path| path.starts_with('/')) + .ok_or_else(|| { + ControlError::Invalid( + "location container requires an absolute working directory".into(), + ) + })?; + let mut command = arguments(&Launcher::Docker(location).scope(), claim); + command.push(project.into()); + command.extend(["--image-id".into(), self.image_id.clone()]); + let mut variables: Vec = parent["Config"]["Env"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|entry| !excluded(entry.split('=').next().unwrap_or(entry), control_variable)) + .map(str::to_owned) + .collect(); + variables.extend( + environment(claim, bearer) + .into_iter() + .map(|(key, value)| format!("{key}={value}")), + ); + let body = json!({ + "Image": self.image_id, "Entrypoint": [location.executable], "Cmd": command, + "WorkingDir": project, "User": parent["Config"]["User"], "Env": variables, + "Labels": {"io.sqlbuild.claim-id": self.claim_id.to_string()}, + "HostConfig": { + "NetworkMode": format!("container:{}", location.location_container), + "VolumesFrom": [location.location_container], + "Memory": parent["HostConfig"]["Memory"], "MemorySwap": parent["HostConfig"]["MemorySwap"], + "NanoCpus": parent["HostConfig"]["NanoCpus"], "PidsLimit": parent["HostConfig"]["PidsLimit"], + "ReadonlyRootfs": parent["HostConfig"]["ReadonlyRootfs"], + }, + }); + self.request( + Method::POST, + &format!("containers/create?name={}", self.name), + Some(&body), + ) + .await? + .error_for_status() + .map_err(docker_error)?; + self.request( + Method::POST, + &format!("containers/{}/start", self.name), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)?; + Ok(()) + } + + pub(in crate::workers) async fn try_wait(&self) -> Result, ControlError> { + let response = self + .request(Method::GET, &format!("containers/{}/json", self.name), None) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(Some(WorkerExit::Absent)); + } + let container: Value = response + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + if container["Config"]["Labels"]["io.sqlbuild.claim-id"] != self.claim_id.to_string() + || container["Image"] != self.image_id + { + return Err(ControlError::Invalid( + "Docker container does not belong to the retained claim and image".into(), + )); + } + if container["State"]["Running"].as_bool() == Some(true) { + return Ok(None); + } + if container["State"]["Status"].as_str() == Some("created") { + return Ok(Some(WorkerExit::Absent)); + } + let code = container["State"]["ExitCode"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("Docker container has no exit evidence".into()))?; + Ok(Some(WorkerExit::Code(code))) + } + + pub(in crate::workers) async fn signal(&self, signal: &str) -> Result<(), ControlError> { + if self.try_wait().await?.is_some() { + return Ok(()); + } + let response = self + .request( + Method::POST, + &format!("containers/{}/kill?signal={signal}", self.name), + None, + ) + .await?; + if !matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::CONFLICT + ) { + response.error_for_status().map_err(docker_error)?; + } + Ok(()) + } + + async fn request( + &self, + method: Method, + path: &str, + body: Option<&Value>, + ) -> Result { + let request = self + .client + .request(method, format!("http://localhost/{path}")); + match body { + Some(body) => request.json(body), + None => request, + } + .send() + .await + .map_err(docker_error) + } +} + +fn docker_error(error: reqwest::Error) -> ControlError { + ControlError::Invalid(format!("Docker daemon request failed: {error}")) +} + +#[cfg(unix)] +fn docker_client(socket: &Path) -> Result { + reqwest::Client::builder() + .no_proxy() + .unix_socket(socket) + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(docker_error) +} + +#[cfg(not(unix))] +fn docker_client(_socket: &Path) -> Result { + Err(ControlError::Invalid( + "Docker launchers require a Unix socket".into(), + )) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs new file mode 100644 index 0000000000..67f3054326 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -0,0 +1,47 @@ +//! Both launchers hand off the same scoped invocation and exclude privileged credentials. + +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::runs::models::RunClaim; + +use crate::workers::models::LauncherScope; + +pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) -> Vec { + [ + "location".into(), + "execute".into(), + "--server".into(), + scope.server_url.to_owned(), + "--location-id".into(), + scope.location_id.to_string(), + "--target".into(), + scope.target.to_owned(), + "--run-id".into(), + claim.request.run_id.to_string(), + "--claim-id".into(), + claim.claim_id.to_string(), + "--project-dir".into(), + ] + .into() +} + +pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { + name == control_variable + || matches!( + name, + "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" + ) + || name == INVOCATION_CONTEXT_ENV +} + +pub(in crate::workers) fn environment( + claim: &RunClaim, + bearer: &str, +) -> [(&'static str, String); 2] { + [ + ("SQLBUILD_RUN_TOKEN", bearer.into()), + (INVOCATION_CONTEXT_ENV, serde_json::json!({ + "run_id": claim.request.run_id, + "integration": {"name": "sqlbuild", "run_id": claim.request.run_id, "job_name": claim.request.job_key}, + }).to_string()), + ] +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs index 16f45af3e7..b182827d2d 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -1,3 +1,7 @@ +mod child; pub(super) mod configuration; -mod local; +mod docker; +mod invocation; +mod recovery; mod shutdown; +mod supervisor; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs new file mode 100644 index 0000000000..1704f88f6b --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -0,0 +1,64 @@ +//! A new leader resumes supervising pinned containers without relaunching their execution. + +use std::collections::BTreeSet; +use std::io::Write; + +use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::models::{ClaimedWorker, DockerWorker, WorkerChild, Workers}; + +impl Workers { + pub(in crate::workers) async fn recover( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if self.recovered_epoch == Some(lease.epoch) { + return Ok(()); + } + let mut known: BTreeSet<_> = self + .active + .iter() + .map(|worker| worker.claim.claim_id) + .collect(); + for location in &self.configuration.docker { + let mut after = None; + loop { + renew_lease(store, lease).await?; + let claims = + list_run_claims(store, (location.location_id, &location.target), after).await?; + let Some(last) = claims.last() else { + break; + }; + after = Some(last.request.run_id); + for claim in claims { + if !known.insert(claim.claim_id) { + continue; + } + let child = match DockerWorker::prepare(location, &claim) { + Ok(child) => Some(WorkerChild::Docker(child)), + Err(ControlError::Invalid(_)) => None, + Err(error) => return Err(error), + }; + drop(writeln!( + std::io::stderr().lock(), + "Worker supervision recovered for run {}", + claim.request.run_id + )); + self.active.push(ClaimedWorker { + claim, + child, + stopping: None, + exit: None, + }); + } + } + } + self.recovered_epoch = Some(lease.epoch); + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs index 24ab13b5cf..9613633420 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -11,10 +11,10 @@ use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; -use crate::workers::_helpers::local::{exit_code, publish, terminal, terminate}; -use crate::workers::models::{LocalWorker, LocalWorkers}; +use crate::workers::_helpers::supervisor::{publish, terminal}; +use crate::workers::models::{ClaimedWorker, WorkerChild, WorkerExit, Workers}; -impl LocalWorker { +impl ClaimedWorker { async fn reap( &mut self, store: &ControlStore, @@ -26,9 +26,11 @@ impl LocalWorker { if self.exit.is_some() { return Ok(()); } + let mut killed = false; if self.stopping.is_none() { - if terminate(child).is_err() { + if child.terminate().await.is_err() { child.kill().await?; + killed = true; } self.stopping = Some(Instant::now()); } @@ -40,10 +42,9 @@ impl LocalWorker { self.exit = Some(exit?); return Ok(()); } - _ = tokio::time::sleep_until(deadline.into()) => { + _ = tokio::time::sleep_until(deadline.into()), if !killed => { child.kill().await?; - self.exit = Some(child.wait().await?); - return Ok(()); + killed = true; } _ = renew.tick(), if lease.is_some() => { if let Some(lease) = lease @@ -66,9 +67,9 @@ impl LocalWorker { if !terminal(state.status) { let lease = lease.ok_or(ControlError::Fenced)?; let reason = if self.child.is_none() { - "local worker launch preparation failed before a child started" + "worker launch preparation failed before a child started" } else { - "scheduler stopped its local worker and reaped the child" + "scheduler confirmed its worker stopped or absent" }; publish( store, @@ -83,7 +84,7 @@ impl LocalWorker { RunOutcome::Failed }, reason: reason.into(), - exit_code: self.exit.map(exit_code), + exit_code: self.exit.and_then(WorkerExit::code), }, ), ) @@ -93,7 +94,7 @@ impl LocalWorker { } } -impl LocalWorkers { +impl Workers { pub(in crate::workers) async fn shutdown_workers( &mut self, store: &ControlStore, @@ -101,9 +102,31 @@ impl LocalWorkers { ) -> Result<(), ControlError> { let mut retrying = false; let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + let mut renewed = Instant::now(); while !self.active.is_empty() { let mut failure = None; for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + if let Err(error) = renew_lease(store, lease).await { + failure = Some(error); + } + renewed = Instant::now(); + } + if let Some(WorkerChild::Docker(child)) = &self.active[index].child { + match child.try_wait().await { + Ok(None) => { + self.active.swap_remove(index); + continue; + } + Ok(exit) => self.active[index].exit = exit, + Err(error) => { + failure.get_or_insert(error); + continue; + } + } + } match self.active[index].finish_shutdown(store, lease).await { Ok(()) => { self.active.swap_remove(index); @@ -121,7 +144,7 @@ impl LocalWorkers { if !retrying { drop(writeln!( std::io::stderr().lock(), - "Retrying worker shutdown publication; stopped workers and their claims remain owned" + "Retrying worker shutdown publication or physical observation; claims remain owned" )); retrying = true; } @@ -140,7 +163,7 @@ impl LocalWorkers { if retrying { drop(writeln!( std::io::stderr().lock(), - "Worker shutdown publication recovered; claims released" + "Worker shutdown recovered; stopped claims released and live containers remain recoverable" )); } Ok(()) diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs similarity index 66% rename from crates/sqlbuild-cli/src/workers/_helpers/local.rs rename to crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 49bbbe75af..b49de36293 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/local.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -1,11 +1,10 @@ -//! The privileged control role owns and reaps local children; workers receive claim-scoped HTTP. +//! One privileged owner supervises both launchers; workers receive only claim-scoped HTTP. use std::io::Write; use std::process::{ExitStatus, Stdio}; use std::time::{Duration, Instant}; use chrono::{TimeDelta, Utc}; -use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; use sqlbuild_orchestration::runs::main::change_run::change_run; @@ -14,25 +13,40 @@ use sqlbuild_orchestration::runs::main::observe_run::observe_run; use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, RunStatus, }; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; use tokio::process::{Child, Command}; use uuid::Uuid; -use crate::workers::models::{LocalLauncher, LocalWorker, LocalWorkers}; +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::models::{ + ClaimedWorker, DockerWorker, Launcher, LocalLauncher, WorkerChild, WorkerExit, Workers, +}; -impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { +impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { async fn poll( &mut self, store: &ControlStore, lease: Option<&SchedulerLease>, ) -> Result<(), ControlError> { + if let Some(lease) = lease { + self.recover(store, lease).await?; + } + let mut renewed = Instant::now(); for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } let worker = &mut self.active[index]; let mut state = observe_run(store, worker.claim.request.run_id).await?; if let Some(child) = worker.child.as_mut() { - worker.exit = worker.exit.or(child.try_wait()?); + worker.exit = worker.exit.or(child.try_wait().await?); } if worker.exit.is_none() && let Some(child) = worker.child.as_mut() @@ -48,7 +62,7 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { if (state.cancel_requested_at.is_some() || not_started || silent) && worker.stopping.is_none() { - terminate(child)?; + child.terminate().await?; worker.stopping = Some(Instant::now()); } if worker @@ -56,7 +70,7 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { .is_some_and(|at| at.elapsed() >= Duration::from_secs(60)) { child.kill().await?; - worker.exit = child.try_wait()?; + worker.exit = child.try_wait().await?; state = observe_run(store, worker.claim.request.run_id).await?; } } @@ -78,9 +92,8 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { "local-exit", RunChange::Finished { outcome, - reason: "local worker exited; physical child has been reaped" - .into(), - exit_code: worker.exit.map(exit_code), + reason: "worker stopped; physical exit or absence confirmed".into(), + exit_code: worker.exit.and_then(WorkerExit::code), }, ), ) @@ -92,28 +105,53 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { let Some(lease) = lease else { return Ok(()); }; - for location in &self.configuration.local { + self.launch_available(store, lease).await + } + + /// Never release claims until children have physically stopped, even if shutdown loses its fence. + async fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.shutdown_workers(store, lease).await + } +} + +impl Workers { + async fn launch_available( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + let mut renewed = Instant::now(); + for location in self.configuration.locations() { + let scope = location.scope(); let running = self .active .iter() .filter(|worker| { - worker.claim.request.location_id == location.location_id - && worker.claim.request.target == location.target + worker.claim.request.location_id == scope.location_id + && worker.claim.request.target == scope.target }) .count(); - for _ in running..location.max_workers.get() { + for _ in running..scope.max_workers.get() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } let Some(claim) = claim_next_run( store, lease, &self.configuration.concurrency_limits, - Some((location.location_id, &location.target)), + Some((scope.location_id, scope.target)), ) .await? else { break; }; let index = self.active.len(); - self.active.push(LocalWorker { + self.active.push(ClaimedWorker { claim, child: None, stopping: None, @@ -121,11 +159,29 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { }); let worker = &mut self.active[index]; let claim = &worker.claim; - if claim.request.image_id.is_some() { + if matches!(location, Launcher::Local(_)) && claim.request.image_id.is_some() { publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; self.active.swap_remove(index); continue; } + let container_id = if let Launcher::Docker(location) = location { + match DockerWorker::prepare(location, claim) { + Ok(child) => { + let name = child.name.clone(); + worker.child = Some(WorkerChild::Docker(child)); + Some(name) + } + Err(_) => { + publish(store, lease, claim, ("docker-image-rejected", RunChange::Finished { + outcome: RunOutcome::Failed, reason: "Docker worker requires a pinned image and supported launcher configuration".into(), exit_code: None, + })).await?; + self.active.swap_remove(index); + continue; + } + } + } else { + None + }; let context = AuditContext { actor: format!("scheduler/{}", lease.holder), granting_role: "scheduler".into(), @@ -146,17 +202,33 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { store, lease, claim, - ("local-launched", RunChange::Launched { container_id: None }), + ("local-launched", RunChange::Launched { container_id }), ) .await?; - match spawn(location, claim, &credential.bearer, &self.control_variable) { - Ok(child) => { + let launched = match location { + Launcher::Local(location) => { + spawn(location, claim, &credential.bearer, &self.control_variable).map( + |child| { + worker.child = Some(WorkerChild::Local(child)); + }, + ) + } + Launcher::Docker(location) => { + if let Some(WorkerChild::Docker(child)) = &worker.child { + child + .launch(location, claim, &credential.bearer, &self.control_variable) + .await?; + } + Ok(()) + } + }; + match launched { + Ok(()) => { drop(writeln!( std::io::stderr().lock(), "Worker launched for run {}", claim.request.run_id )); - worker.child = Some(child); } Err(_) => { publish( @@ -180,15 +252,6 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for LocalWorkers { } Ok(()) } - - /// Never release claims until children have physically stopped, even if shutdown loses its fence. - async fn shutdown( - &mut self, - store: &ControlStore, - lease: Option<&SchedulerLease>, - ) -> Result<(), ControlError> { - self.shutdown_workers(store, lease).await - } } fn spawn( @@ -198,39 +261,15 @@ fn spawn( control_variable: &str, ) -> Result { Ok(Command::new(&location.executable) - .args([ - "location", - "execute", - "--server", - &location.server_url, - "--location-id", - &location.location_id.to_string(), - "--target", - &location.target, - "--run-id", - &claim.request.run_id.to_string(), - "--claim-id", - &claim.claim_id.to_string(), - "--project-dir", - ]) + .args(arguments(&Launcher::Local(location).scope(), claim)) .arg(&location.project_dir) .current_dir(&location.project_dir) - .env_remove(control_variable) - .env_remove("SQLBUILD_CONTROL_DATABASE_URL") - .env_remove("SQLBUILD_LOCATION_TOKEN") - .env("SQLBUILD_RUN_TOKEN", bearer) - .env( - INVOCATION_CONTEXT_ENV, - serde_json::json!({ - "run_id": claim.request.run_id, - "integration": { - "name": "sqlbuild", - "run_id": claim.request.run_id, - "job_name": claim.request.job_key, - }, - }) - .to_string(), - ) + .env_clear() + .envs(std::env::vars_os().filter(|(key, _)| { + key.to_str() + .is_none_or(|key| !excluded(key, control_variable)) + })) + .envs(environment(claim, bearer)) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::inherit()) diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs new file mode 100644 index 0000000000..881c48bea1 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -0,0 +1,3 @@ +//! Immutable Docker image identities use the full SHA-256 hexadecimal digest. + +pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs index abf03f2f04..2c35ad2bd4 100644 --- a/crates/sqlbuild-cli/src/workers/main/mod.rs +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -1 +1 @@ -pub(crate) mod open_local_workers; +pub(crate) mod open_workers; diff --git a/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_workers.rs similarity index 62% rename from crates/sqlbuild-cli/src/workers/main/open_local_workers.rs rename to crates/sqlbuild-cli/src/workers/main/open_workers.rs index 33803a125e..544dae4fc5 100644 --- a/crates/sqlbuild-cli/src/workers/main/open_local_workers.rs +++ b/crates/sqlbuild-cli/src/workers/main/open_workers.rs @@ -1,18 +1,19 @@ -//! Open the installation's local process owner before accepting scheduler work. +//! Open the installation's process owner before accepting scheduler work. use sqlbuild_orchestration::store::errors::ControlError; use crate::workers::_helpers::configuration::read_configuration; -use crate::workers::models::LocalWorkers; +use crate::workers::models::Workers; -pub(crate) fn open_local_workers( +pub(crate) fn open_workers( configuration_path: Option<&str>, control_variable: String, scheduler_enabled: bool, -) -> Result { - Ok(LocalWorkers { +) -> Result { + Ok(Workers { configuration: read_configuration(configuration_path, scheduler_enabled)?, active: Vec::new(), control_variable, + recovered_epoch: None, }) } diff --git a/crates/sqlbuild-cli/src/workers/mod.rs b/crates/sqlbuild-cli/src/workers/mod.rs index ec01b51f5c..59bf71d77a 100644 --- a/crates/sqlbuild-cli/src/workers/mod.rs +++ b/crates/sqlbuild-cli/src/workers/mod.rs @@ -1,5 +1,6 @@ //! Native process owners launch claim-scoped workers without transferring control-store access. mod _helpers; +mod constants; pub(crate) mod main; pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 6fb061ad4d..3cd00b537f 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -3,45 +3,97 @@ use std::collections::BTreeMap; use std::num::{NonZeroU32, NonZeroUsize}; use std::path::PathBuf; -use std::process::ExitStatus; use std::time::Instant; use sqlbuild_orchestration::runs::models::RunClaim; use tokio::process::Child; use uuid::Uuid; +/// Physical ownership and stop evidence are independent of the scheduler's durable run state. +#[derive(Debug)] +pub(super) enum WorkerChild { + Local(Child), + Docker(DockerWorker), +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum WorkerExit { + Code(i64), + Absent, +} + #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct LocalLaunchers { +pub(super) struct Launchers { + #[serde(default)] pub(super) local: Vec, #[serde(default)] + pub(super) docker: Vec, + #[serde(default)] pub(super) concurrency_limits: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(super) struct LauncherScope<'a> { + pub(super) location_id: Uuid, + pub(super) target: &'a str, + pub(super) server_url: &'a str, + pub(super) max_workers: NonZeroUsize, +} + /// No executable, project path or API address is supplied by an untrusted run request. #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct LocalLauncher { pub(super) location_id: Uuid, pub(super) target: String, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, pub(super) project_dir: PathBuf, pub(super) executable: PathBuf, +} + +/// The installation chooses the daemon and location container; the snapshot pins the worker image. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DockerLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, pub(super) server_url: String, pub(super) max_workers: NonZeroUsize, + pub(super) socket: PathBuf, + pub(super) location_container: String, + pub(super) executable: String, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum Launcher<'a> { + Local(&'a LocalLauncher), + Docker(&'a DockerLauncher), +} + +/// Retain the deterministic identity before create/start, including uncertain HTTP responses. +#[derive(Debug)] +pub(super) struct DockerWorker { + pub(super) client: reqwest::Client, + pub(super) name: String, + pub(super) claim_id: Uuid, + pub(super) image_id: String, } #[derive(Debug)] -pub(super) struct LocalWorker { +pub(super) struct ClaimedWorker { pub(super) claim: RunClaim, - pub(super) child: Option, + pub(super) child: Option, pub(super) stopping: Option, - pub(super) exit: Option, + pub(super) exit: Option, } /// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. #[derive(Debug)] -pub(crate) struct LocalWorkers { - pub(super) configuration: LocalLaunchers, - pub(super) active: Vec, +pub(crate) struct Workers { + pub(super) configuration: Launchers, + pub(super) active: Vec, pub(super) control_variable: String, + pub(super) recovered_epoch: Option, } diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 4340e30294..d9635b34d3 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -1,12 +1,13 @@ //! Privileged test operators stay separate from the API-only worker session. use std::collections::BTreeMap; +use std::process::{Command, ExitStatus, Stdio}; use std::time::{Duration, SystemTime}; use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, IssuedToken}; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; @@ -20,6 +21,7 @@ use sqlbuild_orchestration::worker::models::WorkerSession; use uuid::Uuid; use crate::admin_helpers::administer; +use crate::helpers::isolated_binary; use crate::test_types::TestControl; use crate::worker::test_types::WorkerControl; @@ -27,6 +29,30 @@ pub(crate) async fn start_claimed_worker( control: &TestControl, url: &str, ) -> (WorkerSession, WorkerControl) { + let (worker, baseline, token) = claim_worker_run(control, None).await; + change_run( + &worker.store, + RunAuthority::Scheduler(&worker.lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: worker.identity.0, + claim_id: Some(worker.identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("launched worker"); + let client = connect_worker(url, &token.bearer, worker.identity).expect("worker connection"); + let session = start_worker(&client, &baseline) + .await + .expect("verified worker"); + (session, worker) +} + +pub(crate) async fn claim_worker_run( + control: &TestControl, + image_id: Option<&str>, +) -> (WorkerControl, SnapshotInput, IssuedToken) { let path = control.directory.path().join(".sqlbuild/control.sqlite3"); let store = open_store(path.to_str().expect("test database path")) .await @@ -47,7 +73,7 @@ pub(crate) async fn start_claimed_worker( git_revision: None, git_dirty: false, git_author: None, - image_id: None, + image_id: image_id.map(str::to_owned), environment_hash: "orders-environment".into(), sqb_version: env!("CARGO_PKG_VERSION").into(), nodes: BTreeMap::from([( @@ -124,30 +150,16 @@ pub(crate) async fn start_claimed_worker( ) .await .expect("run-scoped token"); - change_run( - &store, - RunAuthority::Scheduler(&lease), - &RunChangeRequest { - event_id: Uuid::now_v7(), - run_id, - claim_id: Some(identity.1), - change: RunChange::Launched { container_id: None }, - }, - ) - .await - .expect("launched worker"); - let client = connect_worker(url, &token.bearer, identity).expect("worker connection"); - let session = start_worker(&client, &baseline) - .await - .expect("verified worker"); ( - session, WorkerControl { store, context, + lease, token_id: token.identity.token_id, identity, }, + baseline, + token, ) } @@ -230,3 +242,81 @@ pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestContro control.child.kill().expect("stop API process"); control.child.wait().expect("reap API process"); } + +pub(crate) fn start_recovery_scheduler( + control: &TestControl, + url: &str, + location_id: Uuid, +) -> TestControl { + let directory = tempfile::tempdir().expect("recovery scheduler logs"); + let configuration = directory.path().join("launchers.json"); + let launcher = serde_json::json!({"docker": [{ + "location_id": location_id, "target": "development", "server_url": url, + "max_workers": 1, "socket": directory.path().join("docker.sock"), + "location_container": "orders-location", "executable": "sqb" + }]}); + std::fs::write(&configuration, launcher.to_string()).expect("Docker scope configuration"); + let stderr = directory.path().join("scheduler.log"); + let child = Command::new(isolated_binary(directory.path())) + .args([ + "scheduler", + "--database-env", + "SQLBUILD_RECOVERY_TEST_STORE", + "--launchers", + ]) + .arg(configuration) + .current_dir(directory.path()) + .env("PATH", "") + .env( + "SQLBUILD_RECOVERY_TEST_STORE", + control.directory.path().join(".sqlbuild/control.sqlite3"), + ) + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("scheduler stderr")) + .spawn() + .expect("replacement scheduler"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn wait_for_worker_log(control: &mut TestControl, expected: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let log = std::fs::read_to_string(&control.stderr).expect("worker supervision log"); + assert!( + control + .child + .try_wait() + .expect("scheduler status") + .is_none(), + "{log}" + ); + let Some(_) = log.find(expected) else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return; + } + panic!("worker supervision exhausted readiness attempts"); + }) + .await + .expect("worker supervision phase completes"); +} + +pub(crate) async fn wait_for_worker_exit(control: &mut TestControl) -> ExitStatus { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let Some(exit) = control.child.try_wait().expect("scheduler exit") else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return exit; + } + panic!("worker supervision exhausted shutdown attempts"); + }) + .await + .expect("shutdown publishes retained terminal evidence") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs new file mode 100644 index 0000000000..0cdf1611ca --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -0,0 +1,79 @@ +//! A committed but unlaunchable claim must not poison every replacement scheduler. + +use std::time::Duration; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + claim_worker_run, start_recovery_scheduler, wait_for_worker_exit, wait_for_worker_log, +}; +use crate::worker::test_types::WorkerRecoveryTestCase; + +#[test] +fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_keep_ownership() { + let test_cases = [ + WorkerRecoveryTestCase { + description: "snapshot without an image pin", + image_id: None, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "snapshot with a malformed image pin", + image_id: Some("sha256:invalid"), + expected_status: "failed", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("recovery test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (worker, baseline, _) = claim_worker_run(&control, test_case.image_id).await; + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable ownership"); + connection + .busy_timeout(Duration::from_secs(5)) + .expect("database contention timeout"); + connection.execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00Z'; + CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events + WHEN NEW.event_type='run_terminated' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", + ).expect("expire the departed leader and interrupt terminal publication"); + let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); + wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; + let held: (i64, i64) = connection.query_row( + "SELECT (SELECT COUNT(*) FROM sqb_write_claims), (SELECT COUNT(*) FROM sqb_run_liveness)", + [], |row| Ok((row.get(0)?, row.get(1)?)), + ).expect("retained claim while publication is unavailable"); + assert_eq!(held, (1, 1), "{}", test_case.description); + connection + .execute_batch("DROP TRIGGER reject_recovery_publication") + .expect("restore terminal publication"); + let exit = wait_for_worker_exit(&mut replacement).await; + assert!( + !exit.success(), + "the original publication failure remains visible" + ); + let finished: (String, i64, i64, i64) = connection.query_row( + "SELECT status, (SELECT COUNT(*) FROM sqb_write_claims), + (SELECT COUNT(*) FROM sqb_run_liveness), + (SELECT COUNT(*) FROM recall_events WHERE event_type='run_terminated') FROM sqb_runs WHERE run_id=?1", + [worker.identity.0.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ).expect("terminal fact and released durable claims"); + assert_eq!( + finished, + (test_case.expected_status.into(), 0, 0, 1), + "{}", + test_case.description + ); + assert!( + std::fs::read_to_string(&replacement.stderr) + .expect("shutdown recovery log") + .contains("Worker shutdown recovered") + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs index 73fb9bf997..54d23d6116 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -6,9 +6,16 @@ pub(crate) struct WorkerHeartbeatTestCase { pub(crate) expected_cancelled: bool, } +pub(crate) struct WorkerRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) image_id: Option<&'static str>, + pub(crate) expected_status: &'static str, +} + pub(crate) struct WorkerControl { pub(crate) store: sqlbuild_orchestration::store::models::ControlStore, pub(crate) context: sqlbuild_orchestration::credentials::models::AuditContext, + pub(crate) lease: sqlbuild_orchestration::scheduler::models::SchedulerLease, pub(crate) token_id: uuid::Uuid, pub(crate) identity: (uuid::Uuid, uuid::Uuid), } diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs index 13b54de371..5bb5be2014 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs @@ -2,5 +2,7 @@ mod helpers; #[path = "test_heartbeat.rs"] mod test_heartbeat; +#[path = "test_recovery.rs"] +mod test_recovery; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs new file mode 100644 index 0000000000..216113dab9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs @@ -0,0 +1,39 @@ +//! Recover ownership from active claims, not stale lifecycle events or elapsed heartbeat guesses. + +use uuid::Uuid; + +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::models::RunClaim; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Keyset pages include claimed-but-unstarted work and never include a neighboring scope. +pub async fn list_run_claims( + store: &ControlStore, + (location_id, target): (Uuid, &str), + after: Option, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT l.worker_instance AS claim_id, r.request_json AS request \ + FROM sqb_run_liveness l JOIN sqb_runs r ON r.run_id=l.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND ($3 IS NULL OR r.run_id>$3) \ + ORDER BY r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::OptionalId(after), + ], + ) + .await?; + let claims = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(claims) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index a7989d799a..7609ff479d 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -2,6 +2,7 @@ pub mod change_run; pub mod claim_next_run; pub mod heartbeat_run; pub(crate) mod job_write_set; +pub mod list_run_claims; pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 43619dfc46..a34df737bc 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; use sqlbuild_orchestration::runs::main::list_runs::list_runs; use sqlbuild_orchestration::runs::models::RunStatus; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; @@ -87,6 +88,31 @@ fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then "{}", test_case.description ); + assert_eq!( + list_run_claims(store, (location, "development"), None) + .await + .expect("recover queued claim"), + vec![claim.clone()], + "recovery includes a committed claim before launch" + ); + assert!( + list_run_claims(store, (location, "development"), Some(claim.request.run_id)) + .await + .expect("page after claimed run") + .is_empty() + ); + assert!( + list_run_claims(store, (other, "development"), None) + .await + .expect("unclaimed neighboring location") + .is_empty() + ); + assert!( + list_run_claims(store, (location, "secondary"), None) + .await + .expect("unclaimed neighboring target") + .is_empty() + ); assert!( claim_next_run( store, @@ -99,6 +125,12 @@ fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then .is_none() ); finish(store, &lease, &claim).await; + assert!( + list_run_claims(store, (location, "development"), None) + .await + .expect("terminal claims are not recoverable") + .is_empty() + ); let next = claim_next_run(store, &lease, &BTreeMap::new(), None) .await .expect("unrestricted claim") diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 6edc268f3f..7b78e9f190 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from pathlib import Path from subprocess import Popen -from typing import Literal, NamedTuple +from typing import Any, Literal, NamedTuple class SnapshotApi(NamedTuple): @@ -24,6 +24,20 @@ class ScheduledWorker(NamedTuple): control_store: Path process: Popen[bytes] log: Path + docker: "DockerLocation | None" = None + + +class DockerLocation(NamedTuple): + image_id: str + container: str + socket: Path + + +class WorkerDeployment(NamedTuple): + kind: str + configuration: dict[str, Any] + publication_arguments: tuple[str, ...] = () + docker: DockerLocation | None = None @dataclass(frozen=True) @@ -86,5 +100,13 @@ class WorkerExecutionTestCase: expected_rows: list[tuple[int]] expected_node_count: int = 1 files: dict[str, str] = field(default_factory=dict) + files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" - launcher: Literal["native", "uncooperative"] = "native" + launcher: Literal["native", "uncooperative", "docker"] = "native" + + +@dataclass(frozen=True) +class WorkerRestartTestCase(WorkerExecutionTestCase): + description: str + shutdown_signal: int = 15 + expected_scheduler_exit: int = 0 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py new file mode 100644 index 0000000000..8d7411a51a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py @@ -0,0 +1 @@ +"""Real container launcher verification.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py new file mode 100644 index 0000000000..58135a9dec --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -0,0 +1,126 @@ +"""A real Docker location with the installed runtime and an isolated synthetic warehouse.""" + +import json +import os +import shutil +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from uuid import uuid4 + +import pytest + +import sqlbuild +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DockerLocation, + WorkerDeployment, +) + +DOCKER_SOCKET: Path = Path(os.environ.get("SQLBUILD_TEST_DOCKER_SOCKET", "/var/run/docker.sock")) +REQUIRES_DOCKER: pytest.MarkDecorator = pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_DOCKER_TESTS") != "1" + and (sys.platform != "linux" or shutil.which("docker") is None or not DOCKER_SOCKET.exists()), + reason="Docker worker tests require a Linux Docker daemon", +) + + +def docker_command(socket: Path, *arguments: str) -> str: + """Keep daemon errors useful without dumping a container's credential-bearing environment.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + ["docker", "--host", f"unix://{socket}", *arguments], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stdout + return result.stdout.strip() + + +@contextmanager +def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + """The worker inherits only these runtime/project mounts, never the store or daemon socket.""" + socket: Path = DOCKER_SOCKET + project: Path = directory / "project" + image: str = os.environ.get("SQLBUILD_TEST_WORKER_IMAGE", "alpine:3.23") + image_id: str = docker_command(socket, "image", "inspect", "--format", "{{.Id}}", image) + name: str = f"sqlbuild-test-location-{uuid4()}" + python: Path = Path(sys.executable) + installations: list[Path] = [Path(sys.prefix), Path(sys.base_prefix)] + while python.is_symlink(): + python = python.parent / python.readlink() + installations.append(python.parent.parent) + mounts: list[str] = [] + for path in filter( + Path.exists, + dict.fromkeys( + [ + Path("/usr"), + Path("/lib"), + Path("/lib64"), + *installations, + Path(sqlbuild.__file__).parent.parent, + ] + ), + ): + mounts.extend(["--volume", f"{path}:{path}:ro"]) + mounts.extend(["--volume", f"{project}:{project}:rw"]) + container: str = docker_command( + socket, + "run", + "--detach", + "--name", + name, + "--network", + "host", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--workdir", + str(project), + "--env", + "HOME=/tmp", + "--env", + "SQLBUILD_WORKER_TEST_VALUE=orders-context", + "--env", + "SQLBUILD_LOCATION_TOKEN=synthetic-location-credential", + "--env", + "SQLBUILD_CONTROL_DATABASE_URL=synthetic-control-store", + "--env", + "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", + *mounts, + image_id, + "/usr/bin/sleep", + "300", + ) + try: + yield WorkerDeployment( + "docker", + {"socket": str(socket), "location_container": container, "executable": executable}, + ("--image-id", image_id), + DockerLocation(image_id, container, socket), + ) + finally: + containers: list[str] = docker_command( + socket, "ps", "--all", "--quiet", "--filter", f"volume={project}" + ).split() + for owned in containers: + _ = (directory / f"docker-{owned}.log").write_text( + docker_command(socket, "logs", owned) + ) + _ = docker_command(socket, "rm", "--force", owned) + + +def worker_container(location: DockerLocation, location_id: str, run_id: str) -> dict[str, object]: + """Return only physical identity/exit evidence, not Docker's secret-bearing configuration.""" + name: str = f"sqlbuild-{location_id}-{run_id}" + result: dict[str, object] = json.loads( + docker_command(location.socket, "inspect", "--format", "{{json .State}}", name) + ) + assert ( + docker_command(location.socket, "inspect", "--format", "{{.Image}}", name) + == location.image_id + ) + return result diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 24c11ac8ff..a91b471edf 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -29,8 +29,12 @@ ClaimedWorker, ScheduledWorker, SnapshotApi, + WorkerDeployment, WorkerExecutionTestCase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + docker_deployment, +) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -78,8 +82,10 @@ def wait_for_generation( return current -def wait_for_log(process: subprocess.Popen[bytes], log: Path, text: str) -> None: - deadline: float = time.monotonic() + 15 +def wait_for_log( + process: subprocess.Popen[bytes], log: Path, text: str, *, timeout: float = 15 +) -> None: + deadline: float = time.monotonic() + timeout while text not in log.read_text(): assert process.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() @@ -217,14 +223,17 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: return result -def native_launcher(directory: Path, executable: str) -> AbstractContextManager[str]: +def native_launcher(directory: Path, executable: str) -> AbstractContextManager[WorkerDeployment]: """Use the installed native launcher without a synthetic child.""" - del directory - return nullcontext(executable) + return nullcontext( + WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": executable} + ) + ) @contextmanager -def uncooperative_launcher(directory: Path, executable: str) -> Iterator[str]: +def uncooperative_launcher(directory: Path, executable: str) -> Iterator[WorkerDeployment]: """Own a synthetic process that ignores TERM, including cleanup after a failed assertion.""" del executable script: Path = directory / "worker" @@ -236,7 +245,9 @@ def uncooperative_launcher(directory: Path, executable: str) -> Iterator[str]: ) script.chmod(0o700) try: - yield str(script) + yield WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": str(script)} + ) finally: with suppress(FileNotFoundError, ProcessLookupError): os.kill(int((directory / "project/worker.pid").read_text()), signal.SIGKILL) @@ -270,13 +281,14 @@ def scheduled_worker( | test_case.files, ) executable: str = str(Path(sys.executable).with_name("sqb")) - launchers: dict[str, Callable[[Path, str], AbstractContextManager[str]]] = { + launchers: dict[str, Callable[[Path, str], AbstractContextManager[WorkerDeployment]]] = { "native": native_launcher, + "docker": docker_deployment, "uncooperative": uncooperative_launcher, } with ( snapshot_api(directory / "control") as api, - launchers[test_case.launcher](directory, executable) as launch_executable, + launchers[test_case.launcher](directory, executable) as deployment, ): published: subprocess.CompletedProcess[str] = subprocess.run( [ @@ -293,6 +305,7 @@ def scheduled_worker( api.url, "--generation", "1", + *deployment.publication_arguments, ], env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, capture_output=True, @@ -301,23 +314,15 @@ def scheduled_worker( check=False, ) assert published.returncode == 0, published.stderr + write_files(project_dir=project, files=test_case.files_after_publication) configuration: Path = directory / "launchers.json" - _ = configuration.write_text( - json.dumps( - { - "local": [ - { - "location_id": api.location_id, - "target": "development", - "project_dir": str(project), - "executable": launch_executable, - "server_url": api.url, - "max_workers": 1, - } - ] - } - ) - ) + launcher: dict[str, Any] = { + "location_id": api.location_id, + "target": "development", + "server_url": api.url, + "max_workers": 1, + } | deployment.configuration + _ = configuration.write_text(json.dumps({deployment.kind: [launcher]})) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) @@ -335,18 +340,17 @@ def scheduled_worker( log, directory / "scheduler.stdout", ) as process: - yield ScheduledWorker(api, project, control_store, process, log) + yield ScheduledWorker(api, project, control_store, process, log, deployment.docker) def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -> dict[str, Any]: """Run/node outcomes, Recall, claims and the physical warehouse agree.""" - api, project, control_store, process, log = worker + api, project, control_store, process, log, _ = worker run: dict[str, Any] = wait_for_run(api, process, log) assert run["node_counts_json"] == {test_case.expected_status: test_case.expected_node_count}, ( log.read_text() ) assert run["job_key"] == "annual_orders" - assert "Worker launched for run" in log.read_text() runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} with urlopen( @@ -357,6 +361,11 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - assert nodes[0]["started_at"] is not None assert nodes[0]["ended_at"] is not None with sqlite3.connect(control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events " + "WHERE invocation_id=? AND event_type='run_launched'", + (f"scheduler/{run['run_id']}",), + ).fetchone() == (1,) contexts: list[tuple[str | None]] = control.execute( "SELECT external_context FROM recall_invocations WHERE command = 'location'" ).fetchall() @@ -371,6 +380,48 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - return run +def running_statement(worker: ScheduledWorker) -> dict[str, Any]: + """The worker has started the real long query, not merely entered its Python interpreter.""" + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) + ) + assert active["status"] == "running", worker.log.read_text() + deadline: float = time.monotonic() + 15 + with sqlite3.connect(worker.control_store) as control: + while not control.execute( + "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", + ("%range(10000000000)%",), + ).fetchone(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + return active + + +def cancel_running_statement(worker: ScheduledWorker) -> None: + """Wait for the real long-running query before exercising the administrative cancel command.""" + active: dict[str, Any] = running_statement(worker) + cancelled: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "cancel-run", + active["run_id"], + "--event-id", + str(uuid4()), + "--reason", + "Cancel the active test statement.", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert cancelled.returncode == 0, cancelled.stderr + + def wait_for_run( api: SnapshotApi, process: subprocess.Popen[bytes], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py new file mode 100644 index 0000000000..a09544ebf1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -0,0 +1,252 @@ +"""A published schedule executes in a real pinned container, not a mocked launcher.""" + +import signal +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerExecutionTestCase, + WorkerRestartTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + REQUIRES_DOCKER, + docker_command, + worker_container, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_log, + wait_for_run, + worker_result, +) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pinned container inherits project context but not privileged credentials", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('37')) t(value)", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + files={ + "orchestration/orders.py": ( + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" + " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " + "'SQLBUILD_SCHEDULER_TEST_STORE'):\n" + " assert name not in os.environ, name\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ) + }, + ), + WorkerExecutionTestCase( + "container warehouse error retains failed node evidence", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_docker_job_when_published_then_a_pinned_worker_executes_and_reports( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert worker.docker is not None + deadline: float = time.monotonic() + 10 + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + while state["Running"]: + assert time.monotonic() < deadline, "Terminal worker did not physically exit" + time.sleep(0.02) + state = worker_container(worker.docker, worker.api.location_id, run["run_id"]) + assert state["ExitCode"] == {"success": 0, "failed": 1}[test_case.expected_status] + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "container refuses changed source before writing", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="docker", + files_after_publication={ + "models/orders.sql": "MODEL (description 'Orders.', materialized table);\nSELECT 41 AS amount\n" + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_pinned_snapshot_when_container_source_drifts_then_no_warehouse_write_is_permitted( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run["node_counts_json"] == {"skipped": 1} + assert not (worker.project / "warehouse.duckdb").exists() + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{run['run_id']}" + assert "node content differs" in docker_command(worker.docker.socket, "logs", name) + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] == 1 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "Docker cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "killed Docker worker preserves its physical exit code", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_killed_then_the_monitor_records_exit_and_releases_claims( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{active['run_id']}" + _ = docker_command(worker.docker.socket, "kill", "--signal", "KILL", name) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] + == 137 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerRestartTestCase( + "graceful scheduler restart retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGTERM, + expected_scheduler_exit=0, + ), + WorkerRestartTestCase( + "abrupt scheduler death retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGKILL, + expected_scheduler_exit=-signal.SIGKILL, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_scheduler_restarts_then_the_same_worker_is_adopted_without_relaunch( + test_case: WorkerRestartTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + before: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, active["run_id"] + ) + worker.process.send_signal(test_case.shutdown_signal) + assert worker.process.wait(timeout=10) == test_case.expected_scheduler_exit, ( + worker.log.read_text() + ) + assert worker_container(worker.docker, worker.api.location_id, active["run_id"])["Running"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + assert ( + worker_container(worker.docker, worker.api.location_id, active["run_id"])["Pid"] + == before["Pid"] + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 9e8dad99f7..8658227d69 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -4,8 +4,6 @@ import os import signal import sqlite3 -import subprocess -import sys import time from pathlib import Path from typing import Any @@ -32,11 +30,11 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, claimed_worker, scheduled_worker, snapshot_api, wait_for_log, - wait_for_run, worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -224,7 +222,7 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", "cancelled", [(0,)], - ) + ), ], ids=lambda case: case.description, ) @@ -232,38 +230,7 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: - active: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) - ) - assert active["status"] == "running", worker.log.read_text() - deadline: float = time.monotonic() + 15 - with sqlite3.connect(worker.control_store) as control: - while not control.execute( - "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", - ("%range(10000000000)%",), - ).fetchone(): - assert worker.process.poll() is None, worker.log.read_text() - assert time.monotonic() < deadline, worker.log.read_text() - time.sleep(0.02) - cancelled: subprocess.CompletedProcess[str] = subprocess.run( - [ - str(Path(sys.executable).with_name("sqb")), - "admin", - "--database-env", - "SQLBUILD_SCHEDULER_TEST_STORE", - "cancel-run", - active["run_id"], - "--event-id", - str(uuid4()), - "--reason", - "Cancel the active test statement.", - ], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - assert cancelled.returncode == 0, cancelled.stderr + cancel_running_statement(worker) run: dict[str, Any] = worker_result(worker, test_case) assert run["status"] == test_case.expected_status, worker.log.read_text() From d777cce965b6005d4244a3a15ac74b154a4e6f4e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:48:02 +0000 Subject: [PATCH 036/138] feat: checkpoint backend groups and pinned reruns (WIP) Preserve the current backend implementation as a remote-review checkpoint. The pull request remains draft and this is not release-ready. Group admission and the four complexity fixes passed targeted real-stack verification. Pinned reruns pass the SQLite/PostgreSQL transaction regression; installed worker rerun coverage and final aggregate gates remain in progress. Six existing Fensu interface findings remain unresolved. --- .../edge_cases/orchestration/nested_groups.py | 7 + .../orchestration/edge_cases/definitions.py | 35 +- .../sqlbuild-cli/src/argv/command_table.json | 73 ++ .../src/native_commands/_helpers/decision.rs | 30 +- .../src/native_commands/main/_run_admin.rs | 5 + .../src/native_commands/models.rs | 3 +- .../src/workers/_helpers/configuration.rs | 9 + .../src/workers/_helpers/supervisor.rs | 25 +- .../src/api/_helpers/snapshots.rs | 21 + .../src/api/main/serve_api.rs | 4 + .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/declarations/_helpers/documents.rs | 32 + .../src/declarations/_helpers/mod.rs | 1 + .../declarations/main/frozen_definitions.rs | 22 + .../declarations/main/group_dependencies.rs | 92 ++ .../src/declarations/main/mod.rs | 2 + .../declarations/main/publish_definitions.rs | 16 +- .../declarations/main/snapshot_definitions.rs | 47 +- .../src/declarations/models.rs | 9 + .../src/runs/_helpers/groups/admission.rs | 186 ++++ .../src/runs/_helpers/groups/capacity.rs | 121 +++ .../src/runs/_helpers/groups/context.rs | 86 ++ .../src/runs/_helpers/groups/lifecycle.rs | 206 +++++ .../src/runs/_helpers/groups/mod.rs | 5 + .../src/runs/_helpers/groups/selection.rs | 139 +++ .../runs/_helpers/{ => lifecycle}/changes.rs | 2 +- .../runs/_helpers/{ => lifecycle}/claims.rs | 57 +- .../runs/_helpers/{ => lifecycle}/events.rs | 2 +- .../src/runs/_helpers/lifecycle/mod.rs | 8 + .../runs/_helpers/{ => lifecycle}/nodes.rs | 2 +- .../runs/_helpers/{ => lifecycle}/output.rs | 2 +- .../_helpers/{ => lifecycle}/projection.rs | 2 +- .../{ => lifecycle}/publish_change.rs | 20 +- .../runs/_helpers/{ => lifecycle}/recall.rs | 2 +- .../src/runs/_helpers/mod.rs | 13 +- .../{job_request.rs => requests/job.rs} | 24 +- .../src/runs/_helpers/requests/mod.rs | 3 + .../{request.rs => requests/publication.rs} | 36 +- .../src/runs/_helpers/requests/rerun.rs | 68 ++ .../src/runs/main/advance_groups.rs | 51 ++ .../src/runs/main/change_run.rs | 6 +- .../src/runs/main/claim_next_run.rs | 71 +- .../src/runs/main/heartbeat_run.rs | 2 +- .../src/runs/main/job_write_set.rs | 8 +- .../src/runs/main/mod.rs | 2 + .../src/runs/main/monitor_runs.rs | 2 +- .../src/runs/main/publish_run_output.rs | 2 +- .../src/runs/main/publish_run_recall.rs | 4 +- .../src/runs/main/queue_run.rs | 65 +- .../src/runs/main/request_job_run.rs | 28 +- .../src/runs/main/request_rerun.rs | 54 ++ .../src/runs/main/request_run.rs | 2 +- .../src/runs/main/run_execution.rs | 3 + .../sqlbuild-orchestration/src/runs/models.rs | 79 ++ .../src/scheduler/_helpers/persistence.rs | 10 +- .../src/scheduler/main/evaluate_trigger.rs | 2 +- .../src/scheduler/models.rs | 1 + .../src/snapshots/_helpers/reading.rs | 19 +- .../src/snapshots/main/current_snapshot.rs | 30 +- .../snapshots/main/current_snapshot_record.rs | 23 + .../src/snapshots/main/mod.rs | 3 + .../snapshots/main/next_remote_generation.rs | 6 +- .../snapshots/main/read_snapshot_documents.rs | 36 + .../src/snapshots/main/read_snapshot_nodes.rs | 36 + .../tests/api/test_contract.rs | 38 + crates/sqlbuild-orchestration/tests/runs.rs | 8 + .../tests/runs/claims/helpers.rs | 72 ++ .../tests/runs/test_claims.rs | 93 ++ .../tests/runs/test_group_admission.rs | 803 ++++++++++++++++++ .../tests/runs/test_groups.rs | 423 +++++++++ .../tests/runs/test_reruns.rs | 301 +++++++ .../tests/runs/test_types.rs | 28 + .../src/bindings/_helpers/cli/dispatch.rs | 55 +- .../_helpers/compiled_project/snapshot.rs | 10 +- .../compiled_project/snapshot_declarations.rs | 143 +++- .../compiled_project/snapshot_execution.rs | 19 +- .../_helpers/location_command/worker.rs | 96 ++- .../native_host/orchestration_declarations.rs | 60 +- .../src/schema/_helpers/mod.rs | 1 - .../src/schema/_helpers/postgres_views.rs | 91 -- .../src/schema/_helpers/relational_views.rs | 235 +++-- .../src/schema/_helpers/rendering.rs | 16 + .../schema/main/control_view_statements.rs | 4 +- src/sqlbuild/orchestration.py | 5 +- .../main/compile_snapshot.py | 4 +- .../orchestration_declarations/models.py | 15 + .../compiler/compiled_project/_test_types.py | 1 + .../publication/_test_types.py | 10 + .../compiled_project/publication/helpers.py | 31 +- .../publication/test_worker.py | 295 +++++++ .../test_orchestration_snapshot.py | 76 +- 91 files changed, 4430 insertions(+), 466 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/nested_groups.py create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/changes.rs (98%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/claims.rs (60%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/events.rs (92%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/nodes.rs (97%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/output.rs (97%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/projection.rs (98%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/publish_change.rs (73%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{ => lifecycle}/recall.rs (97%) rename crates/sqlbuild-orchestration/src/runs/_helpers/{job_request.rs => requests/job.rs} (77%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs rename crates/sqlbuild-orchestration/src/runs/_helpers/{request.rs => requests/publication.rs} (62%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_groups.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_reruns.rs delete mode 100644 crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py new file mode 100644 index 0000000000..35727b2092 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py @@ -0,0 +1,7 @@ +"""A group can contain jobs, but cannot contain another group.""" + +from sqlbuild.orchestration import job, job_group + +orders = job("orders_job", description="Build orders.", select="orders") +daily = job_group("daily_orders", description="Build daily orders.", jobs=[orders]) +weekly = job_group("weekly_orders", description="Unsupported nested membership.", jobs=[daily]) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 3fcca7b42e..af566e9dff 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -1,12 +1,12 @@ """Exercise alias deduplication and cyclic factory containers during native discovery.""" -from sqlbuild.orchestration import job, schedule +from sqlbuild.orchestration import job, job_group, schedule edge_orders = job( "edge_snapshot_orders", description="Refresh only the synthetic edge-case models.", select="tag:edge_case", - settings={"run_tests": False}, + settings={"run_tests": False, "vars": {"edge_delivery_batch": "1"}, "reload": True}, concurrency=2, schedule=schedule( cron="15 2 * * *", @@ -20,3 +20,34 @@ edge_alias = edge_orders edge_factory_results = [[edge_orders]] edge_factory_results.append(edge_factory_results) +edge_group = job_group( + "edge_snapshot_group", + description="Exercise group discovery through shared, cyclic declaration containers.", + jobs=[edge_orders], +) +edge_factory_results.append(edge_group) + +edge_inventory = job( + "edge_snapshot_inventory", + description="Exercise a child limit independent of its parent.", + select="tag:edge_case", + concurrency_groups=("edge_products_limit",), +) +edge_products = job( + "edge_snapshot_products", + description="Exercise the opposing parent and child acquisition order.", + select="tag:edge_case", + concurrency_groups=("edge_inventory_limit",), +) +edge_inventory_group = job_group( + "edge_inventory_group", + description="Complete inventory without reserving its child's product limit.", + jobs=[edge_inventory], + concurrency_groups=("edge_inventory_limit",), +) +edge_products_group = job_group( + "edge_products_group", + description="Wait for safe admission when both installation limits are one.", + jobs=[edge_products], + concurrency_groups=("edge_products_limit",), +) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 08bbcf87c6..ed0917d366 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -448,6 +448,74 @@ ] } }, + { + "name": "rerun-run", + "parser": { + "prog": "sqb admin rerun-run", + "description": "Retry a terminal run using its pinned snapshot and settings, with an atomic security audit.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "SOURCE_RUN_ID", + "required": true, + "help": "terminal source run UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "admin_event_id", + "required": true, + "help": "new run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store_true", + "options": ["--from-failure"], + "dest": "admin_from_failure", + "nargs": 0, + "default": false, + "help": "retry exactly the failed and skipped nodes, excluding successful nodes", + "group": 1 + }, + { + "kind": "store", + "options": ["--priority"], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, { "name": "request-run", "parser": { @@ -800,6 +868,11 @@ "metavar": "request-run", "help": "queue a pinned published job with an atomic security audit" }, + { + "dest": "rerun-run", + "metavar": "rerun-run", + "help": "retry a terminal run or exactly its failed and skipped nodes" + }, { "dest": "cancel-run", "metavar": "cancel-run", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index aae4693e44..c0e69b8143 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -8,7 +8,7 @@ use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; -use sqlbuild_orchestration::runs::models::JobRunRequest; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -89,8 +89,9 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result { - let identity_option = if action.as_deref() == Some("request-run") { + Some("change-trigger" | "cancel-run" | "request-run" | "rerun-run") => { + let identity_option = if matches!(action.as_deref(), Some("request-run" | "rerun-run")) + { "--run-id" } else { "--event-id" @@ -101,7 +102,22 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result value.parse::().map_err(|_| { + ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) + })?, + _ => 0, + }; + if action.as_deref() == Some("rerun-run") { + return Ok(ControlAdmin::Rerun(RerunRequest { + run_id: event_id, + source_run_id: id, + from_failure: flag(namespace, "admin_from_failure"), + reason, + priority, + })); + } let target = text(namespace, "admin_target").unwrap_or_default(); let job_key = text(namespace, "admin_job").unwrap_or_default(); if target.trim().is_empty() || job_key.trim().is_empty() { @@ -109,12 +125,6 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result value.parse::().map_err(|_| { - ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) - })?, - _ => 0, - }; return Ok(ControlAdmin::RequestJobRun(JobRunRequest { run_id: event_id, location_id: id, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 7cfcd0074f..2580c2b194 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -12,6 +12,7 @@ use sqlbuild_orchestration::platform::main::location_work::location_work; use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; +use sqlbuild_orchestration::runs::main::request_rerun::request_rerun; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -51,6 +52,10 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let applied = request_rerun(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } ControlAdmin::RequestLocationWork(request) => { let applied = request_location_work(&store, &context, &request).await?; Ok(json!({"request_id": request.request_id, "applied": applied})) diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 619784b8bd..c8e6917938 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -7,7 +7,7 @@ use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; -use sqlbuild_orchestration::runs::models::JobRunRequest; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use uuid::Uuid; use crate::argv::models::Namespace; @@ -19,6 +19,7 @@ pub(crate) enum ControlAdmin { IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), RequestJobRun(JobRunRequest), + Rerun(RerunRequest), RequestLocationWork(LocationWorkRequest), LocationWork(Uuid), CancelRun { diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs index 7377d5d941..0027b59f29 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -33,6 +33,15 @@ impl Launchers { .map(Launcher::Local) .chain(self.docker.iter().map(Launcher::Docker)) } + + pub(in crate::workers) fn dispatch_scopes(&self) -> BTreeSet<(Uuid, String)> { + self.locations() + .map(|location| { + let scope = location.scope(); + (scope.location_id, scope.target.to_owned()) + }) + .collect() + } } pub(in crate::workers) fn read_configuration( diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index b49de36293..b46fe96bab 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -7,11 +7,12 @@ use std::time::{Duration, Instant}; use chrono::{TimeDelta, Utc}; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::advance_groups::advance_groups; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::main::observe_run::observe_run; use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, RunStatus, + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunDispatch, RunOutcome, RunStatus, }; use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; @@ -33,6 +34,22 @@ impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { lease: Option<&SchedulerLease>, ) -> Result<(), ControlError> { if let Some(lease) = lease { + let eligible = self.configuration.dispatch_scopes(); + for location in self.configuration.locations() { + let scope = location.scope(); + advance_groups( + store, + lease, + ( + &self.configuration.concurrency_limits, + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, + ), + ) + .await?; + } self.recover(store, lease).await?; } let mut renewed = Instant::now(); @@ -125,6 +142,7 @@ impl Workers { lease: &SchedulerLease, ) -> Result<(), ControlError> { let mut renewed = Instant::now(); + let eligible = self.configuration.dispatch_scopes(); for location in self.configuration.locations() { let scope = location.scope(); let running = self @@ -144,7 +162,10 @@ impl Workers { store, lease, &self.configuration.concurrency_limits, - Some((scope.location_id, scope.target)), + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, ) .await? else { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs index e5719f11ce..0b6f56f2af 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -9,6 +9,7 @@ use crate::api::_helpers::authentication::require_location; use crate::api::errors::ApiError; use crate::credentials::models::TokenIdentity; use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::current_snapshot_record::current_snapshot_record; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; use crate::store::models::ControlStore; @@ -54,3 +55,23 @@ pub(crate) async fn current( })?; Ok(Json(current)) } + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = SnapshotRecord), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn metadata( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let record = current_snapshot_record(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + })?; + Ok(Json(record)) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index a08c8a3130..37918480f4 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -37,6 +37,10 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + get(snapshots::metadata), + ) .route( "/api/locations/{location_id}/targets/{target}/runs", get(run_reads::recent), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index b39ad9fd6c..06471e75a9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -15,6 +15,7 @@ use crate::api::_helpers::{ location_work::result, snapshots::publish, snapshots::current, + snapshots::metadata, runs::execution, runs::report, runs::heartbeat, diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs new file mode 100644 index 0000000000..cf9e048bb0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -0,0 +1,32 @@ +//! Decode the same immutable declaration documents at publication and execution boundaries. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::declarations::models::Definitions; +use crate::store::errors::ControlError; + +fn document( + documents: &BTreeMap, + name: &str, +) -> Result, ControlError> { + documents.get(name).map_or_else( + || Ok(BTreeMap::new()), + |value| { + serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) + }, + ) +} + +pub(crate) fn decode_definitions( + documents: &BTreeMap, +) -> Result { + Ok(Definitions { + jobs: document(documents, "jobs")?, + groups: document(documents, "groups")?, + triggers: document(documents, "triggers")?, + }) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index 40c4be79ad..fda8521da8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod documents; pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs new file mode 100644 index 0000000000..5b04c1b88f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs @@ -0,0 +1,22 @@ +//! Runs resolve immutable snapshot documents, never a later deployment's job projections. + +use uuid::Uuid; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::models::Definitions; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn frozen_definitions( + transaction: ControlTransaction, + snapshot_id: Uuid, +) -> Result<(ControlTransaction, Definitions), ControlError> { + let (transaction, documents) = read_snapshot_documents( + transaction, + snapshot_id, + Some(&["jobs", "groups", "triggers"]), + ) + .await?; + Ok((transaction, decode_definitions(&documents)?)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs new file mode 100644 index 0000000000..e896fba0a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -0,0 +1,92 @@ +//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::JobDefinition; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +pub fn group_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> Result>, ControlError> { + if members.is_empty() + || members.iter().any(|name| !jobs.contains_key(name)) + || after + .iter() + .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) + { + return Err(ControlError::Invalid( + "group needs known jobs and dependencies within its membership".into(), + )); + } + let edges = lineage_dependencies(input, jobs, (members, after)); + ensure_acyclic(&edges)?; + Ok(edges) +} + +fn lineage_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> BTreeMap> { + let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for name in members { + for key in &jobs[name].selection { + owners.entry(key).or_default().insert(name); + } + } + let mut edges: BTreeMap> = BTreeMap::new(); + for name in members { + let mut dependencies = after.get(name).cloned().unwrap_or_default(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(node) = input.nodes.get(key) + && let Some(upstream) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { + if let Some(owners) = owners.get(upstream) { + dependencies.extend( + owners + .iter() + .filter(|owner| **owner != name) + .map(|owner| (*owner).to_owned()), + ); + } + pending.push(upstream); + } + } + } + edges.insert(name.clone(), dependencies); + } + edges +} + +fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), ControlError> { + let mut remaining = edges.clone(); + while !remaining.is_empty() { + let ready: BTreeSet = remaining + .iter() + .filter(|(_, deps)| deps.is_empty()) + .map(|(name, _)| name.clone()) + .collect(); + if ready.is_empty() { + return Err(ControlError::Invalid( + "group dependencies contain a cycle".into(), + )); + } + remaining.retain(|name, deps| { + deps.retain(|dependency| !ready.contains(dependency)); + !ready.contains(name) + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index a9078f70e6..d87701b504 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -1,4 +1,6 @@ pub mod change_trigger; +pub(crate) mod frozen_definitions; +pub mod group_dependencies; pub mod list_trigger_events; pub mod list_triggers; pub(crate) mod publish_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 6110788d5e..72e6e9b2db 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -26,13 +26,23 @@ pub(crate) async fn publish_definitions( ) .await?; } - for (name, job) in definitions.jobs { + let jobs = definitions + .jobs + .into_iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .into_iter() + .map(|(name, group)| (name, "group", group.job)), + ); + for (name, kind, job) in jobs { let definition = serde_json::to_value(&job).map_err(|error| ControlError::Invalid(error.to_string()))?; (transaction, _) = query(transaction, - "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,'job',$5,$6)", + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), - Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into())]).await?; + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; for node in job.selection { (transaction, _) = query(transaction, "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index 96987a439c..5966cd18b3 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -1,9 +1,9 @@ //! Validate resolved declarations before acquiring the snapshot publication lock. -use std::collections::BTreeMap; - -use serde::de::DeserializeOwned; +use std::collections::BTreeSet; +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::main::group_dependencies::group_dependencies; use crate::declarations::models::{Definitions, Schedule}; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::parse_cron::parse_cron; @@ -11,25 +11,9 @@ use crate::scheduler::models::CatchUp; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; -fn document( - input: &SnapshotInput, - name: &str, -) -> Result, ControlError> { - input.documents.get(name).map_or_else( - || Ok(BTreeMap::new()), - |value| { - serde_json::from_value(value.clone()) - .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) - }, - ) -} - /// Names, references and resolved selections are checked again at the native trust boundary. pub fn snapshot_definitions(input: &SnapshotInput) -> Result { - let definitions = Definitions { - jobs: document(input, "jobs")?, - triggers: document(input, "triggers")?, - }; + let definitions = decode_definitions(&input.documents)?; for (name, job) in &definitions.jobs { if name.trim().is_empty() || job.description.trim().is_empty() @@ -44,10 +28,31 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); + let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?; + let selection: BTreeSet = group + .jobs + .keys() + .flat_map(|name| definitions.jobs[name].selection.iter().cloned()) + .collect(); + if name.trim().is_empty() + || definitions.jobs.contains_key(name) + || group.job.description.trim().is_empty() + || group.job.selection != selection + || group.jobs != dependencies + || !group.job.settings.is_empty() + { + return Err(ControlError::Invalid(format!( + "group {name} has invalid membership, selection or dependencies" + ))); + } + } for (name, trigger) in &definitions.triggers { if name.trim().is_empty() || trigger.description.trim().is_empty() - || !definitions.jobs.contains_key(&trigger.job_key) + || (!definitions.jobs.contains_key(&trigger.job_key) + && !definitions.groups.contains_key(&trigger.job_key)) { return Err(ControlError::Invalid(format!( "trigger {name} needs a description and a known job" diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 4120b8c9b6..623d581f43 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -50,6 +50,14 @@ pub struct JobDefinition { pub concurrency_groups: BTreeSet, } +/// A virtual parent shares run controls, but only its physical children claim node writers. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct GroupDefinition { + #[serde(flatten)] + pub job: JobDefinition, + pub jobs: BTreeMap>, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Schedule { @@ -80,6 +88,7 @@ pub struct TriggerDefinition { #[derive(Clone, Debug, Default, PartialEq)] pub struct Definitions { pub jobs: BTreeMap, + pub groups: BTreeMap, pub triggers: BTreeMap, } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs new file mode 100644 index 0000000000..f893687365 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs @@ -0,0 +1,186 @@ +//! Reconstruct global parent allocations and remaining child demands in the reservation transaction. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::capacity::{claim_resources, parent_resources}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::models::{ + ConcurrencyResource, GroupCapacity, GroupRun, QueueCandidate, RunKind, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::ControlTransaction; + +pub(crate) async fn group_capacity( + transaction: ControlTransaction, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result<(ControlTransaction, GroupCapacity), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE (r.kind='group' AND r.status IN ('queued','running')) \ + OR (r.status='queued' AND r.cancel_requested_at IS NULL AND l.run_id IS NULL) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id", vec![]).await?; + let runs: Vec = rows.into_iter().map(decode_row).collect::>()?; + let parents: BTreeMap<_, _> = runs + .iter() + .filter(|run| run.request_json.kind == RunKind::Group) + .map(|run| (run.request_json.run_id, run)) + .collect(); + let (transaction, slots) = query(transaction, + "SELECT s.group_key, s.run_id FROM sqb_concurrency_slots s JOIN sqb_runs r ON r.run_id=s.run_id WHERE r.kind='group' AND r.status='running'", vec![]).await?; + let mut capacity = GroupCapacity { + available: limits + .iter() + .map(|(key, limit)| { + ( + ConcurrencyResource::Limit(key.clone()), + i64::from(limit.get()), + ) + }) + .collect(), + held: BTreeMap::new(), + needed: parents.keys().map(|id| (*id, BTreeSet::new())).collect(), + waiting: runs + .iter() + .filter(|run| { + run.state.status == RunStatus::Queued + && run.state.cancel_requested_at.is_none() + && eligible.is_none_or(|scopes| { + scopes.contains(&( + run.request_json.location_id, + run.request_json.target.clone(), + )) + }) + && run.request_json.parent_run_id.is_none_or(|id| { + parents.get(&id).is_some_and(|parent| { + parent.state.status == RunStatus::Running + && parent.state.cancel_requested_at.is_none() + }) + }) + }) + .map(|run| QueueCandidate { + request_json: run.request_json.clone(), + queued_at: run.queued_at, + }) + .collect(), + }; + for (id, parent) in &parents { + if parent.state.status == RunStatus::Running { + let held = capacity.held.entry(*id).or_default(); + held.extend( + parent_resources(&parent.request_json) + .into_iter() + .filter(|resource| matches!(resource, ConcurrencyResource::Overlap(..))), + ); + } + } + for slot in slots { + let id = serde_json::from_value(slot["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let key = slot["group_key"] + .as_str() + .ok_or_else(|| ControlError::Invalid("concurrency claim has no group".into()))?; + capacity + .held + .entry(id) + .or_default() + .insert(ConcurrencyResource::Limit(key.into())); + } + for child in runs + .iter() + .filter(|run| run.request_json.kind != RunKind::Group) + { + let Some(parent) = child + .request_json + .parent_run_id + .and_then(|id| parents.get(&id)) + else { + continue; + }; + if parent.state.cancel_requested_at.is_some() + || (parent.state.status == RunStatus::Queued + && !capacity + .waiting + .iter() + .any(|waiter| waiter.request_json.run_id == parent.request_json.run_id)) + { + continue; + } + let inherited = if parent.state.status == RunStatus::Running { + capacity.held[&parent.request_json.run_id].clone() + } else { + parent_resources(&parent.request_json) + }; + capacity + .needed + .entry(parent.request_json.run_id) + .or_default() + .extend(claim_resources(&child.request_json, Some(&inherited))); + } + for resource in capacity + .held + .values() + .flatten() + .chain(capacity.needed.values().flatten()) + { + if !capacity.available.contains_key(resource) { + match resource { + ConcurrencyResource::Overlap(..) => { + capacity.available.insert(resource.clone(), 1); + } + ConcurrencyResource::Limit(key) => { + return Err(ControlError::Invalid(format!( + "concurrency group {key} has no configured limit" + ))); + } + } + } + } + for resource in capacity.held.values().flatten() { + *capacity.available.entry(resource.clone()).or_insert(0) -= 1; + } + Ok((transaction, capacity)) +} + +/// Give earlier executable contenders first claim, without blocking children that drain parents. +pub(crate) async fn earlier_waiter_blocks( + mut transaction: ControlTransaction, + capacity: &GroupCapacity, + candidate: &QueueCandidate, +) -> Result<(ControlTransaction, bool), ControlError> { + if candidate.request_json.parent_run_id.is_some() { + return Ok((transaction, false)); + } + let footprint = parent_resources(&candidate.request_json); + for older in capacity.earlier_waiters(candidate) { + let inherited = older + .request_json + .parent_run_id + .and_then(|id| capacity.held.get(&id)); + let mut waiting = claim_resources(&older.request_json, inherited); + if candidate.request_json.kind == RunKind::Group { + waiting.extend( + capacity + .needed + .get(&older.request_json.run_id) + .into_iter() + .flatten() + .cloned(), + ); + } + if footprint.is_disjoint(&waiting) { + continue; + } + let ready; + (transaction, ready) = child_ready(transaction, &older.request_json).await?; + if ready { + return Ok((transaction, true)); + } + } + Ok((transaction, false)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs new file mode 100644 index 0000000000..23bc9fb8b2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs @@ -0,0 +1,121 @@ +//! Completion-order safety for persistent parent claims and atomic, temporary child claims. + +use std::cmp::Reverse; +use std::collections::BTreeSet; + +use crate::runs::models::{ConcurrencyResource, GroupCapacity, QueueCandidate, RunRequest}; +use crate::scheduler::models::Overlap; + +pub(crate) fn parent_resources(request: &RunRequest) -> BTreeSet { + let mut resources: BTreeSet<_> = request + .concurrency_groups + .iter() + .cloned() + .map(ConcurrencyResource::Limit) + .collect(); + if let Some(key) = &request.job_key { + resources.insert(ConcurrencyResource::Overlap( + request.location_id, + request.target.clone(), + key.clone(), + )); + } + resources +} + +pub(crate) fn claim_resources( + request: &RunRequest, + inherited: Option<&BTreeSet>, +) -> BTreeSet { + parent_resources(request) + .into_iter() + .filter(|resource| match resource { + ConcurrencyResource::Limit(_) => inherited.is_none_or(|held| !held.contains(resource)), + ConcurrencyResource::Overlap(..) => request.overlap != Overlap::Allow, + }) + .collect() +} + +impl GroupCapacity { + /// Find a completion order without reserving children's independent limits. + pub(crate) fn safe(&self, candidate: &RunRequest) -> bool { + let mut work = self.available.clone(); + let mut held = self.held.clone(); + let resources = parent_resources(candidate); + for resource in &resources { + *work.entry(resource.clone()).or_insert(1) -= 1; + } + held.insert(candidate.run_id, resources); + while !held.is_empty() { + let before = held.len(); + held.retain(|id, resources| { + if self.needed[id] + .iter() + .any(|resource| work.get(resource).copied().unwrap_or(1) <= 0) + { + return true; + } + for resource in resources.iter() { + *work.entry(resource.clone()).or_insert(0) += 1; + } + false + }); + if held.len() == before { + return false; + } + } + true + } + + pub(crate) fn blocks_claim( + &self, + candidate: &QueueCandidate, + waiting: &BTreeSet, + ) -> bool { + let inherited = candidate + .request_json + .parent_run_id + .and_then(|id| self.held.get(&id)); + let mut consumed = claim_resources(&candidate.request_json, inherited); + consumed.extend( + parent_resources(&candidate.request_json) + .into_iter() + .filter(|resource| matches!(resource, ConcurrencyResource::Overlap(..))), + ); + !consumed.is_disjoint(waiting) + } + + /// Only temporary blockers drain ahead of younger work; persistent blockers must let owners finish. + pub(crate) fn record_waiter( + &self, + candidate: &RunRequest, + mut waiting: BTreeSet, + ) -> BTreeSet { + let inherited = candidate.parent_run_id.and_then(|id| self.held.get(&id)); + let resources = claim_resources(candidate, inherited); + if resources.iter().all(|resource| { + self.available + .get(resource) + .is_none_or(|available| *available > 0) + }) { + waiting.extend(resources); + } + waiting + } + + pub(crate) fn earlier_waiters<'a>( + &'a self, + candidate: &'a QueueCandidate, + ) -> impl Iterator { + let rank = |value: &QueueCandidate| { + ( + Reverse(value.request_json.priority), + value.queued_at, + value.request_json.run_id, + ) + }; + self.waiting + .iter() + .filter(move |older| rank(older) < rank(candidate)) + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs new file mode 100644 index 0000000000..be4f644f81 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs @@ -0,0 +1,86 @@ +//! Read group membership and failure policy from the parent's immutable publication. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::declarations::models::OnFailure; +use crate::runs::models::{GroupContext, GroupRun, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn load_group( + transaction: ControlTransaction, + parent_id: Uuid, +) -> Result<(ControlTransaction, GroupContext), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1 AND r.kind='group'", + vec![Parameter::Id(parent_id)]).await?; + let parent: GroupRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.parent_run_id=$1 ORDER BY r.run_id", + vec![Parameter::Id(parent_id)]).await?; + let children = rows.into_iter().map(decode_row).collect::>()?; + let (transaction, definitions) = + frozen_definitions(transaction, parent.request_json.snapshot_id).await?; + let (transaction, failures) = query( + transaction, + "SELECT n.node_key FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND n.status='failed'", + vec![Parameter::Id(parent_id)], + ) + .await?; + let failures: BTreeSet = failures + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + let (transaction, nodes) = read_snapshot_nodes( + transaction, + parent.request_json.snapshot_id, + Some(&failures), + ) + .await?; + let critical_failed = nodes + .values() + .any(|node| node.definition["config"]["critical"].as_bool() == Some(true)); + Ok(( + transaction, + GroupContext { + parent, + children, + definitions, + critical_failed, + }, + )) +} + +impl GroupContext { + pub(crate) fn stopped(&self) -> bool { + self.critical_failed + || self.children.iter().any(|child| { + child.request_json.job_key.as_ref().is_some_and(|name| { + self.definitions.jobs[name].on_failure == OnFailure::StopGroup + && (child + .state + .node_counts_json + .get("failed") + .copied() + .unwrap_or_default() + > 0 + || matches!( + child.state.status, + RunStatus::Failed | RunStatus::PresumedFailed + )) + }) + }) + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs new file mode 100644 index 0000000000..9e7cdb8fe2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs @@ -0,0 +1,206 @@ +//! A virtual parent advances only from durable child evidence under the scheduler's fence. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::admission::{earlier_waiter_blocks, group_capacity}; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::_helpers::lifecycle::claims::reserve_claims; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ + ChangingRun, GroupContext, NodeOutcome, QueueCandidate, RunChange, RunChangeRequest, + RunNodeRecord, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Callers have already checked leadership; the surrounding transaction fences the entire advance. +pub(crate) async fn group_change( + transaction: ControlTransaction, + run_id: Uuid, + change: RunChange, +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(run_id)]).await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let event_id = Uuid::new_v5( + &run_id, + &serde_json::to_vec(&change).map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + publish_change( + transaction, + &RunChangeRequest { + event_id, + run_id, + claim_id: state.worker_instance, + change, + }, + &state, + ) + .await +} + +pub(crate) async fn advance_group( + transaction: ControlTransaction, + parent_id: Uuid, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result { + let (mut transaction, group) = load_group(transaction, parent_id).await?; + let parent = &group.parent; + let children = &group.children; + if parent.state.status.is_terminal() { + return Ok(transaction); + } + let stopped = group.stopped(); + if parent.state.cancel_requested_at.is_some() || stopped { + for child in children { + if !child.state.status.is_terminal() && child.state.cancel_requested_at.is_none() { + transaction = group_change( + transaction, + child.request_json.run_id, + RunChange::CancelRequested { + actor: "system:group".into(), + reason: if stopped { + "group stopped after a child failure" + } else { + "parent group cancellation requested" + } + .into(), + }, + ) + .await?; + } + } + } else if parent.state.status == RunStatus::Queued { + let capacity; + (transaction, capacity) = group_capacity(transaction, limits, eligible).await?; + let blocked; + (transaction, blocked) = earlier_waiter_blocks( + transaction, + &capacity, + &QueueCandidate { + request_json: parent.request_json.clone(), + queued_at: parent.queued_at, + }, + ) + .await?; + if blocked || !capacity.safe(&parent.request_json) { + return Ok(transaction); + } + (transaction, _) = query(transaction, "SAVEPOINT group_claim", vec![]).await?; + let reserved; + (transaction, reserved) = reserve_claims(transaction, &parent.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT group_claim", vec![]).await?; + } else { + transaction = group_change(transaction, parent_id, RunChange::Started).await?; + } + (transaction, _) = query(transaction, "RELEASE SAVEPOINT group_claim", vec![]).await?; + } + if !children.is_empty() + && children + .iter() + .all(|child| child.state.status.is_terminal()) + { + transaction = finish_group(transaction, &group, stopped).await?; + } + Ok(transaction) +} + +async fn finish_group( + transaction: ControlTransaction, + group: &GroupContext, + stopped: bool, +) -> Result { + let parent = &group.parent; + let parent_id = parent.request_json.run_id; + let (mut transaction, rows) = query(transaction, + "SELECT n.* FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id WHERE r.parent_run_id=$1 ORDER BY r.run_id, n.node_key", + vec![Parameter::Id(parent_id)]).await?; + let mut nodes: BTreeMap = BTreeMap::new(); + let rank = |status: &str| match status { + "failed" => 5, + "cancelled" => 4, + "skipped" => 3, + "warn" => 2, + "success" => 1, + _ => 0, + }; + for row in rows { + let node: RunNodeRecord = decode_row(row)?; + if nodes + .get(&node.node_key) + .is_none_or(|previous| rank(&node.status) > rank(&previous.status)) + { + nodes.insert(node.node_key.clone(), node); + } + } + let mut outcomes: BTreeSet = BTreeSet::new(); + for key in &parent.request_json.selection { + let node = nodes + .get(key) + .ok_or_else(|| ControlError::Invalid("group node has no child evidence".into()))?; + outcomes.insert(node.status.clone()); + let outcome: NodeOutcome = serde_json::from_value(serde_json::json!(node.status)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction = group_change( + transaction, + parent_id, + RunChange::NodeFinished { + node_key: key.clone(), + outcome, + rows_affected: node.rows_affected, + error_code: node.error_code.clone(), + skipped_because_of: node.skipped_because_of.clone(), + }, + ) + .await?; + } + let good = outcomes.contains("success") || outcomes.contains("warn"); + let bad = group + .children + .iter() + .any(|child| !matches!(child.state.status, RunStatus::Success | RunStatus::Warn)) + || outcomes + .iter() + .any(|status| !matches!(status.as_str(), "success" | "warn")); + let outcome = if parent.state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else if stopped { + RunOutcome::Failed + } else if bad && good { + RunOutcome::Partial + } else if bad { + RunOutcome::Failed + } else if outcomes.contains("warn") + || group + .children + .iter() + .any(|child| child.state.status == RunStatus::Warn) + { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + group_change( + transaction, + parent_id, + RunChange::Finished { + outcome, + reason: "all group children have durable terminal outcomes".into(), + exit_code: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs new file mode 100644 index 0000000000..9f205ce914 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs @@ -0,0 +1,5 @@ +pub(in crate::runs) mod admission; +pub(in crate::runs) mod capacity; +pub(in crate::runs) mod context; +pub(in crate::runs) mod lifecycle; +pub(in crate::runs) mod selection; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs new file mode 100644 index 0000000000..a42f48821c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -0,0 +1,139 @@ +//! Failed nodes block only their lineage dependents unless an upstream job requests broader stops. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::OnFailure; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::models::{RunRequest, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn child_ready( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, true)); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let dependencies = &group.definitions.groups[parent_name].jobs[name]; + let ready = group.parent.state.status == RunStatus::Running + && group.parent.state.cancel_requested_at.is_none() + && !group.stopped() + && group.children.iter().all(|child| { + child + .request_json + .job_key + .as_ref() + .is_none_or(|name| !dependencies.contains(name)) + || child.state.status.is_terminal() + }); + Ok((transaction, ready)) +} + +pub(crate) async fn skipped_nodes( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, BTreeMap::new())); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let edges = &group.definitions.groups[parent_name].jobs; + let mut ancestors: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); + while let Some(name) = pending.pop() { + if ancestors.insert(name) { + pending.extend(edges[name].iter().map(String::as_str)); + } + } + let (transaction, rows) = query(transaction, + "SELECT n.node_key, n.status, n.skipped_because_of FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND r.run_id<>$2 AND n.status IN ('failed','skipped','cancelled')", + vec![Parameter::Id(parent_id), Parameter::Id(request.run_id)]).await?; + let failures: BTreeMap = rows + .into_iter() + .filter_map(|row| { + let key = row.get("node_key")?.as_str()?.to_owned(); + let cause = row + .get("skipped_because_of") + .and_then(serde_json::Value::as_str) + .unwrap_or(&key) + .to_owned(); + Some((key, cause)) + }) + .collect(); + let whole_job = group.children.iter().find(|child| { + child.request_json.job_key.as_ref().is_some_and(|name| { + ancestors.contains(name.as_str()) + && group.definitions.jobs[name].on_failure == OnFailure::SkipDownstreamJobs + }) && child.state.status.is_terminal() + && !matches!(child.state.status, RunStatus::Success | RunStatus::Warn) + }); + if let Some(child) = whole_job { + let cause = child + .request_json + .selection + .iter() + .find_map(|key| failures.get(key)) + .cloned() + .unwrap_or_else(|| format!("run:{}", child.request_json.run_id)); + return Ok(( + transaction, + request + .selection + .iter() + .map(|key| (key.clone(), cause.clone())) + .collect(), + )); + } + if failures.is_empty() { + return Ok((transaction, BTreeMap::new())); + } + let (transaction, nodes) = read_snapshot_nodes(transaction, request.snapshot_id, None).await?; + let mut skipped: BTreeMap = BTreeMap::new(); + for selected in &request.selection { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending = vec![selected.as_str()]; + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(cause) = failures.get(key) { + skipped.insert(selected.clone(), cause.clone()); + break; + } + if let Some(node) = nodes.get(key) + && let Some(deps) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + pending.extend(deps.iter().filter_map(serde_json::Value::as_str)); + } + } + } + Ok((transaction, skipped)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index 6c9fcee455..8310532774 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -1,4 +1,4 @@ -//! Validate lifecycle authority before publishing any fact or releasing physical write claims. +//! Lifecycle authority checks precede fact publication and physical claim release. use crate::runs::models::{ ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs similarity index 60% rename from crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs index 8e69205b5f..0fcc9bebc1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/claims.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs @@ -1,9 +1,13 @@ -//! A rejected candidate rolls back its entire savepoint, including any partial write set. +//! Reserve write claims and concurrency slots inside the candidate's rollback savepoint. use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroU32; -use crate::runs::models::RunRequest; +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::models::{RunClaim, RunRequest}; use crate::scheduler::models::Overlap; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -21,8 +25,9 @@ pub(crate) async fn reserve_claims( let rows; (transaction, rows) = query( transaction, - "SELECT r.run_id FROM sqb_runs r JOIN sqb_run_liveness l ON l.run_id=r.run_id \ - WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 LIMIT 1", + "SELECT r.run_id FROM sqb_runs r LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND (l.run_id IS NOT NULL OR (r.kind='group' AND r.status='running')) LIMIT 1", vec![ Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), @@ -46,6 +51,18 @@ pub(crate) async fn reserve_claims( } } for group in &request.concurrency_groups { + if let Some(parent_id) = request.parent_run_id { + let inherited; + (transaction, inherited) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 AND run_id=$2", + vec![Parameter::Text(group.clone()), Parameter::Id(parent_id)], + ) + .await?; + if !inherited.is_empty() { + continue; + } + } let limit = limits .get(group) .ok_or_else(|| { @@ -82,3 +99,35 @@ pub(crate) async fn reserve_claims( } Ok((transaction, true)) } + +/// Publish ownership only after every physical claim has been reserved in this transaction. +pub(crate) async fn publish_claim( + transaction: ControlTransaction, + request: RunRequest, +) -> Result<(ControlTransaction, RunClaim), ControlError> { + let claim = RunClaim { + claim_id: Uuid::new_v5(&request.run_id, b"run_claimed"), + request, + }; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", + vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; + let (transaction, cursor) = append_run_event( + transaction, + (claim.request.run_id, claim.claim_id), + "run_claimed", + json!({"claim": claim}), + ) + .await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![ + Parameter::Integer(cursor), + Parameter::Id(claim.request.run_id), + ], + ) + .await?; + Ok((transaction, claim)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs similarity index 92% rename from crates/sqlbuild-orchestration/src/runs/_helpers/events.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs index cb7b38073c..a146a8e4a7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs @@ -1,4 +1,4 @@ -//! All orchestration transitions share Recall's invocation sequence and transaction. +//! Lifecycle facts share Recall's invocation sequence and transaction. use serde_json::Value; use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs new file mode 100644 index 0000000000..0ef9c931db --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -0,0 +1,8 @@ +pub(in crate::runs) mod changes; +pub(in crate::runs) mod claims; +pub(in crate::runs) mod events; +pub(in crate::runs) mod nodes; +pub(in crate::runs) mod output; +pub(in crate::runs) mod projection; +pub(in crate::runs) mod publish_change; +pub(in crate::runs) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs index 8ae44b1a94..6c609d9340 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs @@ -1,4 +1,4 @@ -//! Updating one node adjusts bounded status counters instead of rescanning every node per event. +//! Node lifecycle projections adjust bounded counters instead of rescanning nodes per event. use std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/runs/_helpers/output.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs index beb7a601ef..80f466411a 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/output.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs @@ -1,4 +1,4 @@ -//! Resolve output ownership from the physical claim and normalize Recall input before append. +//! Normalize Recall output under its physical claim's invocation envelope. use chrono::{DateTime, SecondsFormat}; use sqlbuild_recall::records::main::output_row::output_row; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index 9d0629b574..7f6ffb2de0 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use serde_json::json; -use crate::runs::_helpers::nodes::project_node; +use crate::runs::_helpers::lifecycle::nodes::project_node; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs similarity index 73% rename from crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 18ba3f5835..68cc3edd32 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -2,8 +2,8 @@ use serde_json::json; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::_helpers::projection::project_change; +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::lifecycle::projection::project_change; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -22,9 +22,19 @@ pub(crate) async fn publish_change( ) .await?; let mut transaction = project_change(transaction, request, state).await?; - if matches!(request.change, RunChange::CancelRequested { .. }) - && state.worker_instance.is_none() - { + let mut cancel_unstarted = matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none(); + if cancel_unstarted { + let rows; + (transaction, rows) = query( + transaction, + "SELECT kind FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + cancel_unstarted = rows.first().and_then(|row| row["kind"].as_str()) != Some("group"); + } + if cancel_unstarted { let terminal = RunChangeRequest { event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), run_id: request.run_id, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs index 442b0ba521..ecd6740f79 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/recall.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs @@ -1,4 +1,4 @@ -//! Global Recall keys cannot escape a worker's claim namespace. +//! Namespace Recall facts and validate artifact ownership before publication. use std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs index a8a8e0a6f5..5d47d7cf23 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -1,10 +1,3 @@ -pub(crate) mod changes; -pub(crate) mod claims; -pub(crate) mod events; -pub(crate) mod job_request; -pub(crate) mod nodes; -pub(crate) mod output; -pub(crate) mod projection; -pub(crate) mod publish_change; -pub(crate) mod recall; -pub(crate) mod request; +pub(crate) mod groups; +pub(crate) mod lifecycle; +pub(crate) mod requests; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs similarity index 77% rename from crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 691436c7c5..edd8116dd6 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -1,4 +1,4 @@ -//! Freeze execution inputs from one published job under the caller's write transaction. +//! Freeze execution inputs from one published job or group in the caller's write transaction. use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; @@ -15,9 +15,9 @@ pub(crate) async fn frozen_job_request( ) -> Result<(ControlTransaction, RunRequest), ControlError> { let (transaction, jobs) = query( transaction, - "SELECT j.snapshot_id, j.definition_json, s.image_id FROM sqb_jobs j \ + "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ - JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3 AND j.kind='job'", + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], ).await?; let row = jobs.first().ok_or_else(|| { @@ -29,12 +29,10 @@ pub(crate) async fn frozen_job_request( .map_err(|error| ControlError::Invalid(error.to_string()))?; let image_id = serde_json::from_value(row["image_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, write_set) = job_write_set( - transaction, - snapshot_id, - (request.location_id, &request.target, &request.job_key), - ) - .await?; + let kind: RunKind = serde_json::from_value(row["kind"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = + job_write_set(transaction, snapshot_id, &definition.selection).await?; Ok(( transaction, RunRequest { @@ -43,7 +41,7 @@ pub(crate) async fn frozen_job_request( target: request.target.clone(), snapshot_id, image_id, - kind: RunKind::Job, + kind, origin: RunOrigin::Manual, job_key: Some(request.job_key.clone()), parent_run_id: None, @@ -54,7 +52,11 @@ pub(crate) async fn frozen_job_request( requested_by: context.actor.clone(), priority: request.priority, selection: definition.selection, - write_set, + write_set: if kind == RunKind::Group { + Default::default() + } else { + write_set + }, settings: definition.settings, concurrency_groups: definition.concurrency_groups, overlap: definition.overlap, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs new file mode 100644 index 0000000000..010bce9e3a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -0,0 +1,3 @@ +pub(in crate::runs) mod job; +pub(in crate::runs) mod publication; +pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs similarity index 62% rename from crates/sqlbuild-orchestration/src/runs/_helpers/request.rs rename to crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs index b4b727be05..5f6a1eae99 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/request.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs @@ -1,15 +1,44 @@ -//! Validate and publish a frozen request inside the caller's audit transaction. +//! Validate and publish frozen requests inside the caller's audit transaction. use std::collections::BTreeSet; use serde_json::json; +use uuid::Uuid; +use crate::credentials::models::AuditContext; use crate::runs::main::queue_run::queue_run; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Check the original authored request before reading mutable publication pointers or outcomes. +pub(crate) async fn request_retry( + transaction: ControlTransaction, + context: &AuditContext, + (run_id, event_id, request): (Uuid, Uuid, &serde_json::Value), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, previous) = query( + transaction, + "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", + vec![Parameter::Id(event_id), Parameter::Id(run_id)], + ).await?; + let Some(previous) = previous.first() else { + return Ok((transaction, false)); + }; + if previous.get("actor") != Some(&json!(context.actor)) + || previous + .get("after_json") + .and_then(|after| after.get("request")) + != Some(request) + { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + Ok((transaction, true)) +} + pub(crate) async fn publish_request( transaction: ControlTransaction, request: &RunRequest, @@ -36,6 +65,11 @@ pub(crate) async fn publish_request( } return Ok((transaction, false)); } + if request.parent_run_id.is_some() { + return Err(ControlError::Invalid( + "group children are created atomically with their pinned parent".into(), + )); + } let (transaction, nodes) = query( transaction, "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs new file mode 100644 index 0000000000..cd1e9a7ecc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -0,0 +1,68 @@ +//! Retry immutable execution inputs, narrowing only to durable failed and skipped outcomes. + +use std::collections::BTreeSet; + +use serde_json::from_value; + +use crate::credentials::models::AuditContext; +use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_rerun( + transaction: ControlTransaction, + context: &AuditContext, + request: &RerunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT request_json, status FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let source = rows + .first() + .ok_or_else(|| ControlError::Invalid("source run was not found".into()))?; + let status: RunStatus = from_value(source["status"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if !status.is_terminal() { + return Err(ControlError::Invalid( + "rerun requires a terminal source run".into(), + )); + } + let mut frozen: RunRequest = from_value(source["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.from_failure { + let rows; + (transaction, rows) = query( + transaction, + "SELECT node_key FROM sqb_run_nodes WHERE run_id=$1 AND status IN ('failed','skipped')", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let failed: BTreeSet<_> = rows + .iter() + .filter_map(|row| row["node_key"].as_str()) + .collect(); + frozen.selection.retain(|key| failed.contains(key.as_str())); + frozen + .write_set + .retain(|key| frozen.selection.contains(key)); + if frozen.selection.is_empty() { + return Err(ControlError::Invalid( + "source run has no failed or skipped nodes".into(), + )); + } + } + frozen.run_id = request.run_id; + frozen.origin = RunOrigin::Rerun; + frozen.parent_run_id = None; + frozen.root_run_id = request.run_id; + frozen.trigger_id = None; + frozen.tick_id = None; + frozen.rerun_of_run_id = Some(request.source_run_id); + frozen.requested_by = context.actor.clone(); + frozen.priority = request.priority; + Ok((transaction, frozen)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs new file mode 100644 index 0000000000..33b0e9320e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs @@ -0,0 +1,51 @@ +//! Resume virtual parents independently of launcher capacity or process restarts. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; +use std::time::{Duration, Instant}; + +use crate::runs::_helpers::groups::lifecycle::advance_group; +use crate::runs::models::RunDispatch; +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::{check_lease::check_lease, renew_lease::renew_lease}; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn advance_groups<'a>( + store: &ControlStore, + lease: &SchedulerLease, + (limits, dispatch): (&BTreeMap, impl Into>), +) -> Result<(), ControlError> { + let dispatch = dispatch.into(); + let scope = dispatch.scope.ok_or_else(|| { + ControlError::Invalid("group advancement requires a dispatch scope".into()) + })?; + let mut after = None; + let mut renewed = Instant::now(); + loop { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id FROM sqb_runs WHERE kind='group' AND status IN ('queued','running') \ + AND location_id=$1 AND target=$2 AND ($3 IS NULL OR run_id>$3) ORDER BY run_id LIMIT 100", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into()), Parameter::OptionalId(after)]).await?; + commit(transaction).await?; + if rows.is_empty() { + return Ok(()); + } + for row in rows { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let run_id: uuid::Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = + check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let transaction = advance_group(transaction, run_id, limits, dispatch.eligible).await?; + commit(check_lease(transaction, lease).await?).await?; + after = Some(run_id); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 012ff4dbe2..c43086efe7 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -3,8 +3,10 @@ use serde_json::json; use crate::credentials::main::append_audit::append_audit; -use crate::runs::_helpers::changes::{validate_authority, validate_change, validate_retry}; -use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::_helpers::lifecycle::changes::{ + validate_authority, validate_change, validate_retry, +}; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs index 9bf328e7ed..41b5686001 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -1,15 +1,14 @@ //! Claim priority-ordered work without allowing a blocked page to starve later candidates. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroU32; -use serde_json::json; use sqlbuild_recall::schema::models::ColumnType; -use uuid::Uuid; -use crate::runs::_helpers::claims::reserve_claims; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::models::{QueueCandidate, RunClaim}; +use crate::runs::_helpers::groups::admission::{earlier_waiter_blocks, group_capacity}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::_helpers::lifecycle::claims::{publish_claim, reserve_claims}; +use crate::runs::models::{ConcurrencyResource, QueueCandidate, RunClaim, RunDispatch}; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::store::errors::ControlError; @@ -20,13 +19,20 @@ use crate::store::main::query::query; use crate::store::models::{Connection, ControlStore, Parameter, TransactionMode}; /// Limits come from one installation configuration, never from competing per-job declarations. -pub async fn claim_next_run( +pub async fn claim_next_run<'a>( store: &ControlStore, lease: &SchedulerLease, limits: &BTreeMap, - scope: Option<(Uuid, &str)>, + dispatch: impl Into>, ) -> Result, ControlError> { - let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let dispatch = dispatch.into(); + let (mut transaction, capacity) = group_capacity( + check_lease(begin(store, TransactionMode::Write).await?, lease).await?, + limits, + dispatch.eligible, + ) + .await?; + let mut waiting: BTreeSet = BTreeSet::new(); let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { " FOR UPDATE OF r SKIP LOCKED" } else { @@ -36,7 +42,7 @@ pub async fn claim_next_run( loop { let rows; (transaction, rows) = query(transaction, - &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.cancel_requested_at IS NULL \ + &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.kind<>'group' AND r.cancel_requested_at IS NULL \ AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ AND ($4 IS NULL OR (r.location_id=$4 AND r.target=$5)) \ AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ @@ -44,14 +50,27 @@ pub async fn claim_next_run( vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id)), - Parameter::OptionalId(scope.map(|(location, _)| location)), - Parameter::OptionalText(scope.map(|(_, target)| target.to_owned()))]).await?; + Parameter::OptionalId(dispatch.scope.map(|(location, _)| location)), + Parameter::OptionalText(dispatch.scope.map(|(_, target)| target.to_owned()))]).await?; if rows.is_empty() { commit(check_lease(transaction, lease).await?).await?; return Ok(None); } for row in rows { let candidate: QueueCandidate = decode_row(row)?; + let ready; + (transaction, ready) = child_ready(transaction, &candidate.request_json).await?; + if !ready { + after = Some(candidate); + continue; + } + let blocked; + (transaction, blocked) = + earlier_waiter_blocks(transaction, &capacity, &candidate).await?; + if blocked || capacity.blocks_claim(&candidate, &waiting) { + after = Some(candidate); + continue; + } (transaction, _) = query(transaction, "SAVEPOINT candidate_claim", vec![]).await?; let reserved; (transaction, reserved) = @@ -61,34 +80,12 @@ pub async fn claim_next_run( query(transaction, "ROLLBACK TO SAVEPOINT candidate_claim", vec![]).await?; (transaction, _) = query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + waiting = capacity.record_waiter(&candidate.request_json, waiting); after = Some(candidate); continue; } - let claim = RunClaim { - claim_id: Uuid::new_v5(&candidate.request_json.run_id, b"run_claimed"), - request: candidate.request_json, - }; - let now = transaction.now; - (transaction, _) = query(transaction, - "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", - vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; - let cursor; - (transaction, cursor) = append_run_event( - transaction, - (claim.request.run_id, claim.claim_id), - "run_claimed", - json!({"claim": claim}), - ) - .await?; - (transaction, _) = query( - transaction, - "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", - vec![ - Parameter::Integer(cursor), - Parameter::Id(claim.request.run_id), - ], - ) - .await?; + let claim; + (transaction, claim) = publish_claim(transaction, candidate.request_json).await?; (transaction, _) = query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; commit(check_lease(transaction, lease).await?).await?; diff --git a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs index a5399c6cdf..dc0b6db2be 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::runs::_helpers::events::append_run_event; +use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::models::{RunChange, RunChangeRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs index 101f419a95..91840568ee 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -11,12 +11,11 @@ use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn job_write_set( transaction: ControlTransaction, snapshot_id: Uuid, - (location_id, target, job_key): (Uuid, &str, &str), + selection: &BTreeSet, ) -> Result<(ControlTransaction, BTreeSet), ControlError> { let (transaction, rows) = query(transaction, - "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_job_nodes j ON j.node_key=n.node_key \ - WHERE n.snapshot_id=$1 AND j.location_id=$2 AND j.target=$3 AND j.job_key=$4 AND n.node_kind IN ('model','seed','udf','table_fn')", - vec![Parameter::Id(snapshot_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(job_key.into())]).await?; + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND node_kind IN ('model','seed','udf','table_fn')", + vec![Parameter::Id(snapshot_id)]).await?; Ok(( transaction, rows.into_iter() @@ -25,6 +24,7 @@ pub(crate) async fn job_write_set( .and_then(serde_json::Value::as_str) .map(str::to_owned) }) + .filter(|key| selection.contains(key)) .collect(), )) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 7609ff479d..a69509d123 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,3 +1,4 @@ +pub mod advance_groups; pub mod change_run; pub mod claim_next_run; pub mod heartbeat_run; @@ -11,5 +12,6 @@ pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_job_run; +pub mod request_rerun; pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs index b23d4eb791..f6ea6f8420 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs @@ -3,7 +3,7 @@ use chrono::TimeDelta; use uuid::Uuid; -use crate::runs::_helpers::publish_change::publish_change; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; use crate::runs::constants::{HEARTBEAT_TIMEOUT_SECONDS, MONITOR_BATCH_SIZE}; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest}; use crate::scheduler::main::check_lease::check_lease; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs index 8390019aab..17e4e0c7a3 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -3,7 +3,7 @@ use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; -use crate::runs::_helpers::output::{claim_envelope, output_rows}; +use crate::runs::_helpers::lifecycle::output::{claim_envelope, output_rows}; use crate::runs::models::OutputPublication; use crate::store::errors::ControlError; use crate::store::main::append_recall::append_recall; diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs index 9a70513ebd..30c14bb9d8 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -3,8 +3,8 @@ use sqlbuild_recall::store::models::InsertOutcome; use uuid::Uuid; -use crate::runs::_helpers::output::claim_envelope; -use crate::runs::_helpers::recall::{append_records, claim_record}; +use crate::runs::_helpers::lifecycle::output::claim_envelope; +use crate::runs::_helpers::lifecycle::recall::{append_records, claim_record}; use crate::runs::constants::{RECALL_BATCH_ROWS, RECALL_INVOCATION_ID_BYTES}; use crate::runs::models::RecallBatch; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs index 08a7101e43..b8cb2e9b89 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -4,13 +4,74 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::runs::_helpers::events::append_run_event; -use crate::runs::models::RunRequest; +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn queue_run( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let mut transaction = queue_one(transaction, request).await?; + if request.kind == RunKind::Group { + let definitions; + (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; + let group = request + .job_key + .as_ref() + .and_then(|name| definitions.groups.get(name)) + .ok_or_else(|| { + ControlError::Invalid("group is absent from the pinned snapshot".into()) + })?; + let selection_matches = if request.origin == RunOrigin::Rerun { + request.selection.is_subset(&group.job.selection) + } else { + group.job.selection == request.selection + }; + if !selection_matches || !request.write_set.is_empty() { + return Err(ControlError::Invalid( + "group scope differs from its pinned membership".into(), + )); + } + let writers; + (transaction, writers) = + job_write_set(transaction, request.snapshot_id, &request.selection).await?; + for name in group.jobs.keys() { + let job = &definitions.jobs[name]; + let selection: std::collections::BTreeSet<_> = job + .selection + .intersection(&request.selection) + .cloned() + .collect(); + if selection.is_empty() { + continue; + } + let child = RunRequest { + run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), + kind: RunKind::Job, + job_key: Some(name.clone()), + parent_run_id: Some(request.run_id), + rerun_of_run_id: request + .rerun_of_run_id + .map(|id| Uuid::new_v5(&id, name.as_bytes())), + write_set: selection.intersection(&writers).cloned().collect(), + selection, + settings: job.settings.clone(), + concurrency_groups: job.concurrency_groups.clone(), + overlap: job.overlap, + ..request.clone() + }; + transaction = queue_one(transaction, &child).await?; + } + } + Ok(transaction) +} + +async fn queue_one( mut transaction: ControlTransaction, request: &RunRequest, ) -> Result { diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs index e768a67f4d..b54554cb29 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -5,14 +5,13 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::runs::_helpers::job_request::frozen_job_request; -use crate::runs::_helpers::request::publish_request; +use crate::runs::_helpers::requests::job::frozen_job_request; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; use crate::runs::models::JobRunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// The context is a trusted control-store owner, not a role supplied by an HTTP caller. pub async fn request_job_run( @@ -32,22 +31,13 @@ pub async fn request_job_run( } let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = query( + let (transaction, retry) = request_retry( transaction, - "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", - vec![Parameter::Id(event_id), Parameter::Id(request.run_id)], - ).await?; - if let Some(previous) = previous.first() { - if previous.get("actor") != Some(&json!(context.actor)) - || previous - .get("after_json") - .and_then(|after| after.get("request")) - != Some(&json!(request)) - { - return Err(ControlError::Invalid( - "run ID already has different request content".into(), - )); - } + context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { commit(transaction).await?; return Ok(false); } diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs new file mode 100644 index 0000000000..f44d096434 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs @@ -0,0 +1,54 @@ +//! Publish a new pinned execution and its operator audit atomically, including retry identity. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::models::RerunRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// This entry requires direct control-store ownership, just like a privileged manual job request. +pub async fn request_rerun( + store: &ControlStore, + context: &AuditContext, + request: &RerunRequest, +) -> Result { + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "rerun needs an actor, granting role and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"rerun-request"); + let (transaction, retry) = request_retry( + begin(store, TransactionMode::Write).await?, + context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { + commit(transaction).await?; + return Ok(false); + } + let (transaction, frozen) = frozen_rerun(transaction, context, request).await?; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + context, + (event_id, "run.rerun_requested"), + ( + json!({"location_id": frozen.location_id, "target": frozen.target, "job_key": frozen.job_key, "run_id": request.run_id}), + json!({"source_run_id": request.source_run_id}), + json!({"request": request, "frozen": frozen}), + ), + ).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs index 71665e84da..f3041afbbc 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs @@ -1,6 +1,6 @@ //! Queue a frozen, already authorized request without recreating work on transport retries. -use crate::runs::_helpers::request::publish_request; +use crate::runs::_helpers::requests::publication::publish_request; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs index 55c1da3519..f8e49fc1f2 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use uuid::Uuid; +use crate::runs::_helpers::groups::selection::skipped_nodes; use crate::runs::models::{ExecutionMetadata, RunClaim, RunExecution}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -45,6 +46,7 @@ pub async fn run_execution( Ok((fields["node_key"].clone(), fields["node_hash"].clone())) }) .collect::>()?; + let (transaction, skipped_nodes) = skipped_nodes(transaction, &metadata.request_json).await?; commit(transaction).await?; Ok(Some(RunExecution { claim: RunClaim { @@ -54,5 +56,6 @@ pub async fn run_execution( environment_hash: metadata.environment_hash, sqb_version: metadata.sqb_version, node_hashes, + skipped_nodes, })) } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 34444c32e8..c049379a61 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -63,6 +63,16 @@ pub enum RunStatus { PresumedFailed, } +impl RunStatus { + #[must_use] + pub const fn is_terminal(self) -> bool { + !matches!( + self, + Self::Queued | Self::Launched | Self::Running | Self::Unresponsive + ) + } +} + /// The authorized selection and pinned execution inputs are immutable once requested. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRequest { @@ -100,6 +110,17 @@ pub struct JobRunRequest { pub priority: i64, } +/// A retry derives its execution scope from a terminal run, never from caller selectors. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RerunRequest { + pub run_id: Uuid, + pub source_run_id: Uuid, + pub from_failure: bool, + pub reason: String, + pub priority: i64, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, @@ -166,6 +187,9 @@ pub struct RunExecution { pub environment_hash: String, pub sqb_version: String, pub node_hashes: BTreeMap, + /// Derived from completed sibling facts and pinned lineage, never a mutable request rewrite. + #[serde(default)] + pub skipped_nodes: BTreeMap, } #[derive(Debug, Deserialize)] @@ -271,6 +295,28 @@ pub(crate) struct QueueCandidate { pub(crate) queued_at: DateTime, } +/// Fairness covers runnable launcher scopes, not queues the installation cannot dispatch. +#[derive(Clone, Copy, Debug)] +pub struct RunDispatch<'a> { + pub scope: Option<(Uuid, &'a str)>, + pub eligible: Option<&'a BTreeSet<(Uuid, String)>>, +} + +impl<'a> From> for RunDispatch<'a> { + fn from(scope: Option<(Uuid, &'a str)>) -> Self { + Self { + scope, + eligible: None, + } + } +} + +impl<'a> From<(Uuid, &'a str)> for RunDispatch<'a> { + fn from(scope: (Uuid, &'a str)) -> Self { + Some(scope).into() + } +} + #[derive(Clone, Debug, Deserialize)] pub(crate) struct ChangingRun { pub(crate) status: RunStatus, @@ -279,6 +325,39 @@ pub(crate) struct ChangingRun { pub(crate) node_counts_json: BTreeMap, } +/// Virtual parents and physical children use the same lifecycle evidence during group progress. +#[derive(Debug, Deserialize)] +pub(crate) struct GroupRun { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, + #[serde(flatten)] + pub(crate) state: ChangingRun, +} + +/// Parent overlap locks and installation-wide limits occupy different resource namespaces. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ConcurrencyResource { + Limit(String), + Overlap(Uuid, String, String), +} + +/// A transaction-local safety calculation, never a second persisted allocation ledger. +#[derive(Debug)] +pub(crate) struct GroupCapacity { + pub(crate) available: BTreeMap, + pub(crate) held: BTreeMap>, + pub(crate) needed: BTreeMap>, + pub(crate) waiting: Vec, +} + +#[derive(Debug)] +pub(crate) struct GroupContext { + pub(crate) parent: GroupRun, + pub(crate) children: Vec, + pub(crate) definitions: crate::declarations::models::Definitions, + pub(crate) critical_failed: bool, +} + /// Differing retries retain conflict evidence while preserving the first output bytes. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct OutputPublication { diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index e84c4f73b9..3a69b46ea5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -19,7 +19,7 @@ pub(crate) async fn persist_ticks( let (mut transaction, write_set) = job_write_set( transaction, trigger.snapshot_id, - (trigger.location_id, &trigger.target, &trigger.job_key), + &trigger.job_json.selection, ) .await?; for action in &plan.actions { @@ -44,7 +44,7 @@ pub(crate) async fn persist_ticks( target: trigger.target.clone(), snapshot_id: trigger.snapshot_id, image_id: trigger.image_id.clone(), - kind: RunKind::Job, + kind: trigger.run_kind, origin: RunOrigin::Schedule, job_key: Some(trigger.job_key.clone()), parent_run_id: None, @@ -55,7 +55,11 @@ pub(crate) async fn persist_ticks( requested_by: "system:scheduler".into(), priority: 0, selection: trigger.job_json.selection.clone(), - write_set: write_set.clone(), + write_set: if trigger.run_kind == RunKind::Group { + Default::default() + } else { + write_set.clone() + }, settings: trigger.job_json.settings.clone(), concurrency_groups: trigger.job_json.concurrency_groups.clone(), overlap: trigger.job_json.overlap, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs index 1a2e98d3b7..7d065991ee 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs @@ -24,7 +24,7 @@ pub async fn evaluate_trigger( ) -> Result, ControlError> { let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; let (transaction, rows) = query(transaction, - "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, \ + "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, j.kind AS run_kind, \ state.started_at, state.last_tick_for, state.status FROM sqb_triggers t \ JOIN sqb_trigger_state state ON state.trigger_id=t.trigger_id JOIN sqb_snapshots s ON s.snapshot_id=t.snapshot_id \ JOIN sqb_jobs j ON j.location_id=t.location_id AND j.target=t.target AND j.job_key=t.job_key \ diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 9f4d421736..39594ed147 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -125,6 +125,7 @@ pub(crate) struct DueTrigger { pub(crate) image_id: Option, pub(crate) definition_json: TriggerDefinition, pub(crate) job_json: JobDefinition, + pub(crate) run_kind: crate::runs::models::RunKind, pub(crate) started_at: DateTime, pub(crate) last_tick_for: Option>, pub(crate) status: TriggerStatus, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs index 91d5c5f6b0..a350003e29 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -4,12 +4,29 @@ use std::collections::BTreeMap; use serde::de::DeserializeOwned; use serde_json::Value; +use uuid::Uuid; use crate::snapshots::_helpers::content::decompress; -use crate::snapshots::models::SnapshotNode; +use crate::snapshots::models::{SnapshotNode, SnapshotRecord}; use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; +pub(crate) async fn read_current_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + fn decode_body(value: &Value) -> Result { serde_json::from_slice(&decompress(value)?) .map_err(|error| ControlError::Invalid(error.to_string())) diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs index b668008c38..81889f713b 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -2,8 +2,10 @@ use uuid::Uuid; -use crate::snapshots::_helpers::reading::{attach_sources, read_documents, read_nodes}; -use crate::snapshots::models::{CurrentSnapshot, SnapshotChange, SnapshotRecord}; +use crate::snapshots::_helpers::reading::{attach_sources, read_current_record}; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::models::{CurrentSnapshot, SnapshotChange}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -17,27 +19,21 @@ pub async fn current_snapshot( location_id: Uuid, target: &str, ) -> Result, ControlError> { - let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query(transaction, - "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ - FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", - vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; - let Some(row) = rows.into_iter().next() else { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + let Some(record) = record else { return Ok(None); }; - let record: SnapshotRecord = decode_row(row)?; - let (transaction, rows) = query(transaction, - "SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1 ORDER BY n.node_key", - vec![Parameter::Id(record.snapshot_id)]).await?; - let nodes = read_nodes(rows)?; + let (transaction, nodes) = read_snapshot_nodes(transaction, record.snapshot_id, None).await?; let (transaction, rows) = query(transaction, "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash WHERE s.snapshot_id=$1", vec![Parameter::Id(record.snapshot_id)]).await?; let nodes = attach_sources(nodes, rows)?; - let (transaction, rows) = query(transaction, - "SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1", - vec![Parameter::Id(record.snapshot_id)]).await?; - let documents = read_documents(rows)?; + let (transaction, documents) = + read_snapshot_documents(transaction, record.snapshot_id, None).await?; let (transaction, rows) = query(transaction, "SELECT change_kind, subject_key, before_hash, after_hash FROM sqb_snapshot_changes WHERE snapshot_id=$1 ORDER BY change_seq", vec![Parameter::Id(record.snapshot_id)]).await?; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs new file mode 100644 index 0000000000..58f44ee9a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs @@ -0,0 +1,23 @@ +//! Reconcile publication identity without reading node definitions, source bodies or documents. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_current_record; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn current_snapshot_record( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 56f1b75755..6918426ec2 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,5 +1,8 @@ pub mod current_snapshot; +pub mod current_snapshot_record; pub mod next_remote_generation; pub mod publish_remote_snapshot; pub mod publish_snapshot; +pub(crate) mod read_snapshot_documents; +pub(crate) mod read_snapshot_nodes; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs index f92dabe4fa..3c4f974037 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::api::errors::ClientError; use crate::api::main::client_response::client_response; use crate::api::main::connect_client::connect_client; -use crate::snapshots::models::CurrentSnapshot; +use crate::snapshots::models::SnapshotRecord; pub async fn next_remote_generation( server_url: &str, @@ -21,10 +21,10 @@ pub async fn next_remote_generation( "targets", target, "snapshots", + "metadata", ])?; - match client_response::(connection.client.get(url)).await { + match client_response::(connection.client.get(url)).await { Ok(current) => current - .record .generation .checked_add(1) .ok_or(ClientError::Configuration( diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs new file mode 100644 index 0000000000..45a03182f8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs @@ -0,0 +1,36 @@ +//! Decode immutable documents within the caller's consistent snapshot transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_documents; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_documents( + transaction: ControlTransaction, + snapshot_id: Uuid, + kinds: Option<&[&str]>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if kinds.is_some_and(<[&str]>::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = kinds.map_or(String::new(), |keys| { + parameters.extend(keys.iter().map(|key| Parameter::Text((*key).into()))); + format!( + " AND d.kind IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1{filter}"), + parameters).await?; + Ok((transaction, read_documents(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs new file mode 100644 index 0000000000..33240705d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs @@ -0,0 +1,36 @@ +//! Read retained node definitions without expanding their display-only source bodies. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_nodes; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_nodes( + transaction: ControlTransaction, + snapshot_id: Uuid, + selection: Option<&BTreeSet>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if selection.is_some_and(BTreeSet::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = selection.map_or(String::new(), |keys| { + parameters.extend(keys.iter().cloned().map(Parameter::Text)); + format!( + " AND n.node_key IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1{filter} ORDER BY n.node_key"), + parameters).await?; + Ok((transaction, read_nodes(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 9b4b904f29..be3b784298 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -16,6 +16,7 @@ use sqlbuild_orchestration::runs::models::{ RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, }; use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; @@ -108,10 +109,47 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("first generation"), + 1 + ); let input = snapshot(location_id); let published = publish_remote_snapshot(&api.url, &location_token.bearer, &input) .await .expect("publish retained native data"); + let metadata: SnapshotRecord = client + .get(format!("{path}/metadata")) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("metadata response") + .json() + .await + .expect("metadata record without project bodies"); + assert_eq!(metadata, published); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("reconcile published generation"), + 2 + ); + assert!(matches!( + next_remote_generation(&api.url, &location_token.bearer, other, "development") + .await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); let response = client .get(&path) .bearer_auth(&location_token.bearer) diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs index b2b4e61852..df23471f89 100644 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ b/crates/sqlbuild-orchestration/tests/runs.rs @@ -1,3 +1,5 @@ +#[path = "runs/claims/helpers.rs"] +mod claim_helpers; #[path = "store/faults/helpers.rs"] mod fault_helpers; #[path = "store/faults/test_types.rs"] @@ -24,11 +26,17 @@ mod test_claims; mod test_coalesce; #[path = "runs/test_faults.rs"] mod test_faults; +#[path = "runs/test_group_admission.rs"] +mod test_group_admission; +#[path = "runs/test_groups.rs"] +mod test_groups; #[path = "runs/test_lifecycle.rs"] mod test_lifecycle; #[path = "runs/test_monitor.rs"] mod test_monitor; #[path = "runs/test_requests.rs"] mod test_requests; +#[path = "runs/test_reruns.rs"] +mod test_reruns; #[path = "runs/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs new file mode 100644 index 0000000000..4fe22bb22f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -0,0 +1,72 @@ +//! Real slot accounting and successful completion through the public worker lifecycle. + +use std::collections::BTreeMap; + +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, +}; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { + let sql = "SELECT group_key, count(*) FROM sqb_concurrency_slots GROUP BY group_key"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect SQLite") + .prepare(sql) + .expect("slot query") + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("SQLite slot counts") + .collect::>() + .expect("decode slot counts"); + }; + sqlx_core::query_as::query_as::<_, (String, i64)>(sql) + .fetch_all(pool) + .await + .expect("PostgreSQL slot counts") + .into_iter() + .collect() +} + +pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { + let mut changes = vec![RunChange::Started]; + changes.extend( + claim + .request + .selection + .iter() + .map(|node| RunChange::NodeFinished { + node_key: node.clone(), + outcome: NodeOutcome::Success, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + }), + ); + changes.push(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "synthetic child completed".into(), + exit_code: Some(0), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("worker fact"); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index a34df737bc..ad747388df 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; +use crate::claim_helpers::succeed; use crate::helpers::{queue, request, snapshot}; use crate::location_helpers::create_location; use crate::replica_helpers::second_store; @@ -439,3 +440,95 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in }); } } + +#[test] +fn given_a_blocked_multi_limit_waiter_when_younger_work_arrives_then_shared_limits_drain_without_blocking_unrelated_work() + { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite protects priority across atomic limit sets", + backend: Backend::Sqlite, + expected_unblocked_run: 4, + }, + GroupLimitTestCase { + description: "PostgreSQL protects priority across atomic limit sets", + backend: Backend::Postgres, + expected_unblocked_run: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + for (id, priority, nodes, groups) in [ + (1, 100, &["inventory"][..], &["inventory_limit"][..]), + ( + 2, + 10, + &["orders"][..], + &["orders_limit", "inventory_limit"][..], + ), + (3, 1, &["customers"][..], &["orders_limit"][..]), + (4, 0, &["products"][..], &[][..]), + ] { + let mut input = request(scope, id, priority, nodes); + input.concurrency_groups = groups.iter().map(|key| (*key).into()).collect(); + queue(store, &input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let blocker = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("highest priority"); + assert_eq!(blocker.request.run_id, Uuid::from_u128(1)); + let unrelated = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("unrelated"); + assert_eq!( + unrelated.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("wait") + .is_none() + ); + succeed(store, &blocker).await; + let protected = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("older waiter"); + assert_eq!(protected.request.run_id, Uuid::from_u128(2)); + succeed(store, &protected).await; + let younger = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("younger retry"); + assert_eq!(younger.request.run_id, Uuid::from_u128(3)); + succeed(store, &younger).await; + succeed(store, &unrelated).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs new file mode 100644 index 0000000000..8dc2a9e113 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -0,0 +1,803 @@ +//! Real database admission preserves completion orders, capacities and fair retries across replicas. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::{ + advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RunDispatch, RunStatus}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::claim_helpers::{slot_counts, succeed}; +use crate::helpers::{queue, request, snapshot}; +use crate::location_helpers::create_location; +use crate::replica_helpers::second_store; +use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{ + AdmissionParent, GroupAdmissionTestCase, GroupQueueTestCase, MixedAdmissionTestCase, + RunLifecycleTestCase, +}; + +#[test] +fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queue_order_win() { + let test_cases = [ + MixedAdmissionTestCase { + description: "higher-priority physical request precedes parent admission", + physical_priority: 10, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "earlier physical request wins at equal priority", + physical_priority: 1, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "higher-priority parent precedes physical request", + physical_priority: 1, + parent_priority: 10, + expected_selections: ["orders", "inventory"], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "concurrency_groups": ["warehouse"], "jobs": {"stage": []}}})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let mut physical = request( + (location, published.snapshot_id), + 900, + test_case.physical_priority, + &["inventory"], + ); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(100), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise mixed queue ordering".into(), + priority: test_case.parent_priority, + }, + ) + .await + .expect("queue group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + for index in 0..test_case.expected_selections.len() { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + assert_eq!( + claim.request.selection, + BTreeSet::from([test_case.expected_selections[index].into()]), + "{} {backend:?}", + test_case.description + ); + assert_eq!(slot_counts(store).await.get("warehouse"), Some(&1)); + succeed(store, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + assert!( + list_runs(store, scope, 10) + .await + .expect("runs") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknown_child_limits_do_not_block_it() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite dispatch excludes unserved prospective demand", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL dispatch excludes unserved prospective demand", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let unserved = create_location(store, "inventory", "test-admin") + .await + .expect("unserved location"); + let served = create_location(store, "orders", "test-admin") + .await + .expect("served location"); + for location in [unserved, served] { + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": if location == unserved {vec!["inventory_limit"]} else {vec![]}}})); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "jobs": {"stage": []}}})); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: location, + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise launcher eligibility".into(), + priority: 1, + }, + ) + .await + .expect("queue group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let eligible = BTreeSet::from([(served, "development".into())]); + let dispatch = RunDispatch { + scope: Some((served, "development")), + eligible: Some(&eligible), + }; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("eligible parent advances"); + let claim = claim_next_run(store, &lease, &limits, dispatch) + .await + .expect("eligible dispatch") + .expect("eligible child"); + assert_eq!(claim.request.location_id, served); + succeed(store, &claim).await; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("finish eligible parent"); + assert!( + list_runs(store, (served, "development"), 10) + .await + .expect("served runs") + .iter() + .all(|run| run.status == test_case.expected_status), + "{}", + test_case.description + ); + assert!( + list_runs(store, (unserved, "development"), 10) + .await + .expect("unserved runs") + .iter() + .all(|run| run.status == RunStatus::Queued) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_then_every_group_finishes_within_limits() + { + let test_cases = [ + GroupAdmissionTestCase { + description: "capacity two permits both sides of a cycle", + limits: &[("orders_limit", 2), ("inventory_limit", 2)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[&[("orders_limit", 1), ("inventory_limit", 1)]], + }, + GroupAdmissionTestCase { + description: "asymmetric capacity still has a safe completion order", + limits: &[("orders_limit", 2), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "three-way cycle leaves one parent waiting", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + AdmissionParent { + holds: &["products_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "a free first stage does not conceal a later cycle", + limits: &[("orders_limit", 1), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&[], &["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&[], &["orders_limit"]], + }, + ], + expected_running: 1, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "a persistently blocked child lets its parent's peer drain shared limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "multiple parent limits do not hoist independent child limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ("support_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit", "products_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["support_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let mut jobs = serde_json::Map::new(); + let mut groups = serde_json::Map::new(); + let mut child_count = 0; + for (index, parent) in test_case.parents.iter().enumerate() { + let mut members = serde_json::Map::new(); + let mut selection = BTreeSet::new(); + for (stage, needs) in parent.children.iter().enumerate() { + let name = format!("orders_job_{index}_{stage}"); + let node = + ["orders", "customers", "inventory", "products"][child_count % 4]; + child_count += 1; + selection.insert(node); + jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); + let after: Vec = (0..stage) + .map(|earlier| format!("orders_job_{index}_{earlier}")) + .collect(); + members.insert(name, json!(after)); + } + groups.insert(format!("orders_group_{index}"), json!({"description": "Complete a synthetic group", "selection": selection, "concurrency_groups": parent.holds, "jobs": members})); + } + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + let published = publish_snapshot(store, &input) + .await + .expect("publish groups"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits: BTreeMap = test_case + .limits + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive capacity"), + ) + }) + .collect(); + let scope = (location, "development"); + for index in 0..test_case.parents.len() { + request_job_run( + store, + &actor, + &JobRunRequest { + // Higher-priority arrivals exercise safety, not only an older waiter's priority barrier. + run_id: Uuid::from_u128(1000 - index as u128), + location_id: location, + target: "development".into(), + job_key: format!("orders_group_{index}"), + reason: "Exercise capacity-aware admission".into(), + priority: index as i64 + 1, + }, + ) + .await + .expect("request group"); + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler admits higher-priority arrival"); + second.expect("replica cannot admit an unsafe arrival"); + } + for _ in 0..test_case.parents.len() { + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler advance"); + second.expect("second scheduler advance"); + } + let initial = list_runs(store, scope, 100).await.expect("initial runs"); + assert_eq!( + initial + .iter() + .filter(|run| run.status == RunStatus::Running) + .count(), + test_case.expected_running, + "{} {backend:?}", + test_case.description + ); + let before = slot_counts(store).await; + for reduction in test_case.reductions { + let reduced = reduction + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive reduction"), + ) + }) + .collect(); + assert!( + claim_next_run(&replica, &lease, &reduced, Some(scope)) + .await + .expect("unsafe parent state must not fail physical dispatch") + .is_none() + ); + assert_eq!( + slot_counts(store).await, + before, + "blocked work cannot mutate claims" + ); + let unrelated = + request((location, published.snapshot_id), 777, 1, &["products"]); + queue(store, &unrelated).await; + let claim = claim_next_run(&replica, &lease, &reduced, Some(scope)) + .await + .expect("unrelated recovery progress") + .expect("unrelated work still dispatches"); + assert_eq!(claim.request.run_id, unrelated.run_id); + succeed(store, &claim).await; + } + for iteration in 0..child_count { + let owner = [store, &replica][iteration % 2]; + for _ in 0..test_case.parents.len() { + advance_groups(owner, &lease, (&limits, scope)) + .await + .expect("resume group from durable state"); + } + let claim = claim_next_run(owner, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + // Re-evaluate while a physical child still holds its temporary claims. + advance_groups(&replica, &lease, (&limits, scope)) + .await + .expect("advance during running claim"); + let occupied = slot_counts(store).await; + assert!( + occupied + .iter() + .all(|(key, count)| *count <= i64::from(limits[key].get())), + "{} {occupied:?}", + test_case.description + ); + succeed(owner, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("aggregate final children"); + let finished = list_runs(&replica, scope, 100) + .await + .expect("finished groups"); + assert_eq!( + finished.len(), + test_case.parents.len() + child_count + test_case.reductions.len() + ); + assert!( + finished.iter().all(|run| run.status == RunStatus::Success), + "{} {finished:?}", + test_case.description + ); + assert!( + slot_counts(store).await.is_empty(), + "terminal parents release their own slots" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_then_overlap_allow_still_runs() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves overlap allow across generations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves overlap allow across generations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build a pinned stage", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Allow independent generations", "selection": ["orders"], "overlap": "allow", "concurrency_groups": ["orders_limit"], "jobs": {"stage": []}}})); + let first = publish_snapshot(store, &input) + .await + .expect("first snapshot"); + let older = JobRunRequest { + run_id: Uuid::from_u128(2000), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Older group".into(), + priority: 1, + }; + request_job_run(store, &actor, &older) + .await + .expect("older request"); + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["stage"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["concurrency_groups"] = + json!(["inventory_limit"]); + publish_snapshot(store, &input) + .await + .expect("second snapshot"); + let newer = JobRunRequest { + run_id: Uuid::from_u128(1000), + reason: "Independent replacement generation".into(), + ..older.clone() + }; + request_job_run(store, &actor, &newer) + .await + .expect("newer request"); + let mut physical = request((location, first.snapshot_id), 3000, 10, &["products"]); + physical.concurrency_groups.insert("orders_limit".into()); + queue(store, &physical).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let scope = (location, "development"); + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("physical claim") + .expect("higher priority blocker"); + assert_eq!(blocker.request.run_id, physical.run_id); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("independent group admission"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == older.run_id) + .expect("older") + .status, + RunStatus::Queued + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == newer.run_id) + .expect("newer") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("new child") + .expect("allow generation can progress"); + assert_eq!(child.request.parent_run_id, Some(newer.run_id)); + assert_eq!(child.request.selection, ["inventory".to_owned()].into()); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish replacement parent"); + succeed(store, &blocker).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("old parent retries after blocker drains"); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("old child") + .expect("older group makes progress"); + assert_eq!(child.request.parent_run_id, Some(older.run_id)); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete all groups"); + assert!( + list_runs(store, scope, 100) + .await + .expect("finished") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelated_work_has_a_bounded_write_lock() + { + let test_cases = [ + GroupQueueTestCase { + description: "SQLite scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Sqlite, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + GroupQueueTestCase { + description: "PostgreSQL scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Postgres, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let template = input.nodes["orders"].clone(); + for index in 0..test_case.node_count { + let mut node = template.clone(); + node.definition["description"] = + json!(format!("Orders {index} {}", "details ".repeat(128))); + input.nodes.insert(format!("orders_{index}"), node); + } + let jobs = (0..test_case.group_count).map(|index| (format!("job_{index}"), json!({"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]}))).collect::>(); + let groups = (0..test_case.group_count).map(|index| (format!("group_{index}"), json!({"description": "Independent order group", "selection": ["orders"], "jobs": {format!("job_{index}"): []}}))).collect::>(); + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + input.documents.insert( + "columns".into(), + json!({"orders": "unused display metadata ".repeat(100000)}), + ); + let published = publish_snapshot(store, &input) + .await + .expect("publish large project"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + let mut physical = request((location, published.snapshot_id), 1, 10, &["products"]); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("blocker"); + for index in 0..test_case.group_count { + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(1000 + index as u128), + location_id: location, + target: "development".into(), + job_key: format!("group_{index}"), + reason: "Exercise blocked group queue".into(), + priority: 1, + }, + ) + .await + .expect("request group"); + } + let started = std::time::Instant::now(); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit independent parents"); + eprintln!( + "{:?}: {} parent admissions in {:?}", + test_case.backend, + test_case.group_count, + started.elapsed() + ); + let unrelated = request((location, published.snapshot_id), 2, -1, &["customers"]); + queue(store, &unrelated).await; + let started = std::time::Instant::now(); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("scan blocked children") + .expect("unrelated work"); + let elapsed = started.elapsed(); + eprintln!( + "{:?}: {} groups / {} nodes, claim transaction {:?}", + test_case.backend, test_case.group_count, test_case.node_count, elapsed + ); + assert_eq!( + claim.request.run_id, unrelated.run_id, + "{}", + test_case.description + ); + assert!( + elapsed.as_millis() < test_case.expected_max_claim_millis, + "{}: {elapsed:?}", + test_case.description + ); + succeed(store, &claim).await; + succeed(store, &blocker).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs new file mode 100644 index 0000000000..6a502d207d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -0,0 +1,423 @@ +//! Pinned groups coordinate real transactions, inherited slots, cancellation and replay. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, +}; +use sqlbuild_orchestration::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::{ + migrate_store::migrate_store, read_recall_events::read_recall_events, +}; +use uuid::Uuid; + +use crate::claim_helpers::{slot_counts, succeed}; +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::location_helpers::create_location; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; + +#[test] +fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_and_parent_waits_for_terminal_evidence() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite group cancellation and replay", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL group cancellation and replay", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, + "marts": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"]} + }})); + let published = publish_snapshot(store, &input) + .await + .expect("publish group"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build the group".into(), + priority: 1, + }; + assert!( + request_job_run(store, &context, &request) + .await + .expect("request group") + ); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry group") + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one slot"))]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start virtual parent"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + assert_eq!(first.request.job_key.as_deref(), Some("staging")); + assert_eq!(first.request.parent_run_id, Some(request.run_id)); + assert_eq!(first.request.write_set, ["customers".to_owned()].into()); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("dependency waits") + .is_none() + ); + let authority = RunAuthority::Worker { + run_id: first.request.run_id, + claim_id: first.claim_id, + }; + for change in [ + RunChange::Started, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(11), + error_code: None, + skipped_because_of: None, + }, + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "staging completed".into(), + exit_code: Some(0), + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: first.request.run_id, + claim_id: Some(first.claim_id), + change, + }, + ) + .await + .expect("staging fact"); + } + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = + json!(["customers", "inventory"]); + publish_snapshot(store, &input) + .await + .expect("redeploy different membership"); + let second = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim marts") + .expect("marts ready"); + assert_eq!(second.request.snapshot_id, published.snapshot_id); + assert_eq!(second.request.selection, ["orders".to_owned()].into()); + assert_eq!(second.request.write_set, second.request.selection); + assert!( + run_execution(store, (second.request.run_id, second.claim_id)) + .await + .expect("execution") + .expect("pinned") + .skipped_nodes + .is_empty() + ); + let authority = RunAuthority::Worker { + run_id: second.request.run_id, + claim_id: second.claim_id, + }; + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start marts"); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: request.run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: "Cancel the group".into(), + }, + }, + ) + .await + .expect("cancel parent"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("propagate cancellation"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + RunStatus::Running, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == second.request.run_id) + .expect("active child") + .status, + RunStatus::Running + ); + let standalone = JobRunRequest { + run_id: Uuid::now_v7(), + job_key: "staging".into(), + ..request.clone() + }; + request_job_run(store, &context, &standalone) + .await + .expect("standalone request"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("parent retains its slot") + .is_none() + ); + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "worker confirmed its executor stopped".into(), + exit_code: Some(143), + }, + }, + ) + .await + .expect("physical child finished"); + let fault = TransactionFault { + description: "group terminal publication rolls back", + table: "recall_events", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected group publication failure')", + postgres_body: "RAISE EXCEPTION 'injected group publication failure'", + }; + install_fault(store, &fault).await; + assert!( + advance_groups(store, &lease, (&limits, scope)) + .await + .is_err() + ); + remove_fault(store, &fault).await; + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry group completion"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + test_case.expected_status + ); + let standalone_claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("slot released") + .expect("standalone staging ready"); + assert_eq!(standalone_claim.request.run_id, standalone.run_id); + assert_eq!(standalone_claim.request.parent_run_id, None); + let events = read_recall_events(store, 0, 1000).await.expect("facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated" + && event["invocation_id"] == format!("scheduler/{}", request.run_id)) + .count(), + 1 + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups_finish_without_exceeding_limits() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite opposing group reservations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL opposing group reservations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + let mut completed = Vec::new(); + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders_job": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["inventory_limit"]}, + "inventory_job": {"description": "Build inventory", "selection": ["customers"], "concurrency_groups": ["orders_limit"]} + })); + input.documents.insert("groups".into(), json!({ + "orders_group": {"description": "Orders group", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"orders_job": []}}, + "inventory_group": {"description": "Inventory group", "selection": ["customers"], "concurrency_groups": ["inventory_limit"], "jobs": {"inventory_job": []}} + })); + publish_snapshot(store, &input) + .await + .expect("publish groups"); + for job_key in ["orders_group", "inventory_group"] { + request_job_run( + store, + &context, + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: job_key.into(), + reason: "Build independent groups".into(), + priority: 1, + }, + ) + .await + .expect("request group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one slot")), + ( + "inventory_limit".into(), + NonZeroU32::new(1).expect("one slot"), + ), + ]); + let scope = (location, "development"); + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance groups"); + } + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!(runs.len(), 4, "{}", test_case.description); + assert_eq!( + runs.iter() + .filter(|run| run.status == test_case.expected_status) + .count(), + 1, + "{}", + test_case.description + ); + for _ in 0..2 { + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim child") + .expect("a group must make progress"); + let slots = slot_counts(store).await; + assert_eq!( + slots.len(), + 2, + "child-only slot is acquired only for its worker" + ); + assert!(slots.values().all(|occupied| *occupied == 1), "{slots:?}"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("held child") + .is_none() + ); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete parent and retry waiting group"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry independent of run-id ordering"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert!( + runs.iter().all(|run| run.status == RunStatus::Success), + "{runs:?}" + ); + completed.push(test_case.description); + close_test_store(fixture).await; + }); + } + assert_eq!(completed.len(), 2, "both backends must run"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs new file mode 100644 index 0000000000..a17b54cee9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -0,0 +1,301 @@ +//! Terminal retries retain their snapshot and independent limits without repeating successful work. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::json; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, +}; +use sqlbuild_orchestration::runs::models::{ + JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, + RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::claim_helpers::{slot_counts, succeed}; +use crate::fault_helpers::{install_fault, remove_fault}; +use crate::helpers::snapshot; +use crate::location_helpers::create_location; +use crate::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_test_types::Backend; +use crate::test_types::{RunLifecycleTestCase, TransactionFault}; + +#[test] +fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite pinned group rerun", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL pinned group rerun", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input + .nodes + .get_mut("inventory") + .expect("inventory") + .definition["deps"] = json!(["orders"]); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, + "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, + "completed": {"description": "Build products", "selection": ["products"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"], "completed": []} + }})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build orders".into(), + priority: 1, + }, + ) + .await + .expect("request group"); + let retry = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_id, + from_failure: true, + reason: "Retry failed orders".into(), + priority: 7, + }; + assert!( + request_rerun(store, &context, &retry).await.is_err(), + "a live source cannot be rerun" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("warehouse".into(), std::num::NonZeroU32::new(1).unwrap()), + ( + "inventory_limit".into(), + std::num::NonZeroU32::new(1).unwrap(), + ), + ]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit source"); + for _ in 0..3 { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("ready child"); + let mut changes = vec![RunChange::Started]; + for node in &claim.request.selection { + let outcome = match node.as_str() { + "products" => NodeOutcome::Success, + "orders" => NodeOutcome::Failed, + "customers" => NodeOutcome::Cancelled, + _ => NodeOutcome::Skipped, + }; + changes.push(RunChange::NodeFinished { + node_key: node.clone(), + outcome, + rows_affected: None, + error_code: None, + skipped_because_of: (outcome == NodeOutcome::Skipped) + .then(|| "orders".into()), + }); + } + changes.push(RunChange::Finished { + outcome: if claim.request.selection.contains("products") { + RunOutcome::Success + } else { + RunOutcome::Failed + }, + reason: "synthetic source outcome".into(), + exit_code: Some(1), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("source worker fact"); + } + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish source"); + assert_eq!( + list_runs(store, scope, 20) + .await + .unwrap() + .iter() + .find(|run| run.run_id == source_id) + .unwrap() + .status, + RunStatus::Partial + ); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").unwrap()["staging"]["settings"] = + json!({"vars": {"batch": 91}}); + publish_snapshot(store, &input).await.expect("redeploy"); + let fault = TransactionFault { + description: "rerun audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_rerun(store, &context, &retry).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_runs(store, scope, 20).await.unwrap().len(), + 4, + "audit failure rolls back parent and all children" + ); + let (first, again) = tokio::join!( + request_rerun(store, &context, &retry), + request_rerun(store, &context, &retry) + ); + assert_ne!(first.expect("rerun"), again.expect("replica retry")); + let changed = RerunRequest { + from_failure: false, + ..retry.clone() + }; + assert!(request_rerun(store, &context, &changed).await.is_err()); + let other = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!(request_rerun(store, &other, &retry).await.is_err()); + for request in [ + retry.clone(), + RerunRequest { + run_id: Uuid::now_v7(), + from_failure: false, + ..retry.clone() + }, + ] { + if !request.from_failure { + assert!(request_rerun(store, &context, &request).await.unwrap()); + } + let expected: BTreeSet = if request.from_failure { + ["orders", "inventory"].map(str::to_owned).into() + } else { + ["orders", "inventory", "customers", "products"] + .map(str::to_owned) + .into() + }; + let runs = list_runs(store, scope, 20).await.unwrap(); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .unwrap() + .selection_json, + expected + ); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit rerun"); + let mut selected = BTreeSet::new(); + for _ in 0..if request.from_failure { 2 } else { 3 } { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim rerun") + .expect("ready retry child"); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!( + claim.request.image_id, + Some("sha256:synthetic-orders-image".into()) + ); + assert_eq!(claim.request.origin, RunOrigin::Rerun); + assert_eq!(claim.request.priority, 7); + if claim.request.job_key.as_deref() == Some("staging") { + assert_eq!( + claim.request.settings, + BTreeMap::from([("vars".into(), json!({"batch": 23}))]) + ); + } + selected.extend(claim.request.selection.iter().cloned()); + succeed(store, &claim).await; + } + assert_eq!(selected, expected); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish rerun"); + assert_eq!( + list_runs(store, scope, 20) + .await + .unwrap() + .iter() + .find(|run| run.run_id == request.run_id) + .unwrap() + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert!(slot_counts(store).await.is_empty()); + assert!( + !request_rerun(store, &context, &request) + .await + .expect("completed transport retry") + ); + let successful = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: request.run_id, + from_failure: true, + ..request.clone() + }; + assert!( + request_rerun(store, &context, &successful).await.is_err(), + "successful work has nothing to retry from failure" + ); + } + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 0a21b7f2ef..9ee50b5f8c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -62,3 +62,31 @@ pub(crate) struct MonitorBacklogTestCase { pub(crate) run_count: usize, pub(crate) expected_batches: [usize; 3], } + +pub(crate) struct AdmissionParent { + pub(crate) holds: &'static [&'static str], + pub(crate) children: &'static [&'static [&'static str]], +} + +pub(crate) struct GroupAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) limits: &'static [(&'static str, u32)], + pub(crate) parents: &'static [AdmissionParent], + pub(crate) expected_running: usize, + pub(crate) reductions: &'static [&'static [(&'static str, u32)]], +} + +pub(crate) struct GroupQueueTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) group_count: usize, + pub(crate) node_count: usize, + pub(crate) expected_max_claim_millis: u128, +} + +pub(crate) struct MixedAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) physical_priority: i64, + pub(crate) parent_priority: i64, + pub(crate) expected_selections: [&'static str; 2], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 17c4dea5e1..7c0ebc870f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -76,7 +76,7 @@ const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ ]; /// One invocation being dispatched: its namespace and the values derived from it once. -struct Invocation<'py> { +pub(crate) struct Invocation<'py> { args: Bound<'py, PyAny>, project_dir: Bound<'py, PyAny>, select: Bound<'py, PyTuple>, @@ -84,6 +84,27 @@ struct Invocation<'py> { } impl<'py> Invocation<'py> { + pub(crate) fn new(args: &Bound<'py, PyAny>) -> PyResult { + let raw_project_dir = native_field(args, "project_dir")?; + let project_dir = if raw_project_dir.is_none() { + raw_project_dir + } else { + path_class(args.py())? + .call1((raw_project_dir,))? + .call_method0("resolve")? + }; + let (select, selector_files) = selector_inputs(args)?; + if native_field(args, "select_file")?.is_truthy()? { + withhold_launch_record(args.py())?; + } + Ok(Self { + args: args.clone(), + project_dir, + select, + selector_files, + }) + } + fn py(&self) -> Python<'py> { self.args.py() } @@ -97,7 +118,7 @@ impl<'py> Invocation<'py> { } /// The value of one handler argument or request field. - fn value(&self, source: Source) -> PyResult> { + pub(crate) fn value(&self, source: Source) -> PyResult> { let py: Python<'py> = self.py(); match source { Source::Attr(name) => self.attr(name), @@ -425,24 +446,7 @@ fn routed_command<'py>( recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); - let raw_project_dir: Bound<'py, PyAny> = native_field(args, "project_dir")?; - let project_dir: Bound<'py, PyAny> = if raw_project_dir.is_none() { - raw_project_dir - } else { - path_class(py)? - .call1((raw_project_dir,))? - .call_method0("resolve")? - }; - let (select, selector_files) = selector_inputs(args)?; - if native_field(args, "select_file")?.is_truthy()? { - withhold_launch_record(py)?; - } - let invocation = Invocation { - args: args.clone(), - project_dir, - select, - selector_files, - }; + let invocation = Invocation::new(args)?; match command_dispatch(&command_facts(args)?) { Dispatch::Nothing => 0.into_bound_py_any(py), Dispatch::Fail(failure) => Err(user_error(py, failure)), @@ -473,6 +477,12 @@ pub(crate) fn dispatch_command<'py>( handlers: Option>, recall: &InvocationRecall, ) -> PyResult> { + within(&warehouse_scope(args)?, |_| { + routed_command(args, handlers.as_ref(), recall) + }) +} + +pub(crate) fn warehouse_scope<'py>(args: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = args.py(); let warehouse: Bound<'py, PyAny> = native_field(args, "warehouse")?; if !warehouse.is_none() { @@ -491,7 +501,7 @@ pub(crate) fn dispatch_command<'py>( ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; } - let scope: Bound<'py, PyAny> = Bound::new( + Ok(Bound::new( py, command_warehouse_scope( &native_field(args, "command")?, @@ -499,8 +509,7 @@ pub(crate) fn dispatch_command<'py>( Some(&native_field(args, "vars")?), )?, )? - .into_any(); - within(&scope, |_| routed_command(args, handlers.as_ref(), recall)) + .into_any()) } /// The parsed values that select a route, read from the namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index a2f44bf10a..0d95fcf560 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -15,13 +15,13 @@ use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_re use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - AuthoredJob, compile_declarations, discover_declarations, + AuthoredDeclaration, compile_declarations, discover_declarations, }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::native_host::orchestration_declarations::validate_jobs; +use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::records::fields::field; @@ -126,11 +126,11 @@ pub(crate) fn build_orchestration_snapshot( None => discover_declarations(py, &project_dir)?, }; if !declarations.is_empty() { - validate_jobs(py, &declarations)?; + let declarations = collect_declarations(py, declarations)?; let authored = declarations .iter() - .map(|declaration| declaration.extract::()) - .collect::>>()?; + .map(|declaration| declaration.extract::()) + .collect::>>()?; let graph = project.lineage_graph(py)?; input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 54ad885b83..d5922477fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -14,9 +14,10 @@ use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use sqlbuild_discovery::tree::main::rglob::rglob; use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - JobDefinition, OnFailure, Schedule, TriggerDefinition, + GroupDefinition, JobDefinition, OnFailure, Schedule, TriggerDefinition, }; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -83,23 +84,79 @@ pub(crate) struct AuthoredJob { targets: Vec, } +#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] +struct AuthoredJobReference { + name: String, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredGroup { + name: String, + description: String, + jobs: Vec, + after: BTreeMap>, + overlap: String, + concurrency_groups: Vec, + schedule: Option, + targets: Vec, +} + +#[derive(FromPyObject)] +pub(crate) enum AuthoredDeclaration { + Job(AuthoredJob), + Group(AuthoredGroup), +} + +impl AuthoredSchedule { + fn compile(self, name: &str, description: &str) -> PyResult { + let catch_up = if self.max_catch_up.is_null() { + Value::String(self.catch_up) + } else { + serde_json::json!({self.catch_up: {"max_runs": self.max_catch_up}}) + }; + Ok(TriggerDefinition { + description: description.into(), + job_key: name.into(), + schedule: Schedule::Cron { expression: self.cron, timezone: self.timezone }, + starts_at: serde_json::from_value(self.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(self.default_status)) + .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, + }) + } +} + pub(crate) fn compile_declarations( mut input: SnapshotInput, graph: &ProjectGraph, - declarations: Vec, + declarations: Vec, ) -> PyResult { if declarations.is_empty() { return Ok(input); } - if input.documents.contains_key("jobs") || input.documents.contains_key("triggers") { + if ["jobs", "groups", "triggers"] + .iter() + .any(|key| input.documents.contains_key(*key)) + { return Err(PyValueError::new_err( "authored declarations cannot overwrite snapshot job or trigger documents", )); } let mut jobs: BTreeMap = BTreeMap::new(); + let mut authored_groups: Vec = Vec::new(); + let mut authored_jobs: Vec = Vec::new(); + for declaration in declarations { + match declaration { + AuthoredDeclaration::Job(job) => authored_jobs.push(job), + AuthoredDeclaration::Group(group) => authored_groups.push(group), + } + } let mut triggers: BTreeMap = BTreeMap::new(); let mut names: BTreeSet = BTreeSet::new(); - for job in declarations { + for job in authored_jobs { if !names.insert(job.name.clone()) { return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); } @@ -134,31 +191,9 @@ pub(crate) fn compile_declarations( .map(|(kind, name)| format!("{kind}:{name}")) .collect(); if let Some(schedule) = job.schedule { - let catch_up = if schedule.max_catch_up.is_null() { - Value::String(schedule.catch_up) - } else { - serde_json::json!({schedule.catch_up: {"max_runs": schedule.max_catch_up}}) - }; triggers.insert( job.name.clone(), - TriggerDefinition { - description: job.description.clone(), - job_key: job.name.clone(), - schedule: Schedule::Cron { - expression: schedule.cron, - timezone: schedule.timezone, - }, - starts_at: serde_json::from_value(schedule.starts_at) - .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, - default_status: serde_json::from_value(Value::String(schedule.default_status)) - .map_err(|error| { - PyValueError::new_err(format!("default_status: {error}")) - })?, - catch_up: serde_json::from_value(catch_up) - .map_err(|error| PyValueError::new_err(format!( - "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" - )))?, - }, + schedule.compile(&job.name, &job.description)?, ); } jobs.insert( @@ -175,8 +210,62 @@ pub(crate) fn compile_declarations( }, ); } + let mut groups: BTreeMap = BTreeMap::new(); + for group in authored_groups { + if !names.insert(group.name.clone()) { + return Err(PyValueError::new_err(format!( + "duplicate job or group {}", + group.name + ))); + } + if !group.targets.is_empty() && !group.targets.contains(&input.target) { + continue; + } + let members: BTreeSet = group.jobs.iter().map(|job| job.name.clone()).collect(); + if members.len() != group.jobs.len() { + return Err(PyValueError::new_err("group members must be distinct")); + } + let mut after: BTreeMap> = BTreeMap::new(); + for (job, dependencies) in group.after { + after.insert( + job.name, + dependencies + .into_iter() + .map(|dependency| dependency.name) + .collect(), + ); + } + let dependencies = group_dependencies(&input, &jobs, (&members, &after)) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let selection: BTreeSet = members + .iter() + .flat_map(|name| jobs[name].selection.iter().cloned()) + .collect(); + if let Some(schedule) = group.schedule { + triggers.insert( + group.name.clone(), + schedule.compile(&group.name, &group.description)?, + ); + } + groups.insert( + group.name, + GroupDefinition { + job: JobDefinition { + description: group.description, + selection, + settings: BTreeMap::new(), + overlap: serde_json::from_value(Value::String(group.overlap)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + on_failure: OnFailure::SkipDependents, + concurrency_groups: group.concurrency_groups.into_iter().collect(), + }, + jobs: dependencies, + }, + ); + } for (key, document) in [ ("jobs", serde_json::to_value(jobs)), + ("groups", serde_json::to_value(groups)), ("triggers", serde_json::to_value(triggers)), ] { input.documents.insert( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 57d0f43e90..a7aa2f894c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -23,11 +23,24 @@ pub(crate) struct WorkerBuild { impl WorkerBuild { pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { - Ok(Arc::new(Self { + let state = Arc::new(Self { execution: worker.get().execution(worker.py())?, worker: worker.clone().unbind(), completed: Mutex::default(), - })) + }); + for key in state.execution.skipped_nodes.keys() { + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, None))?; + } + Ok(state) + } + + pub(crate) fn has_work(&self) -> bool { + self.execution + .claim + .request + .selection + .iter() + .any(|key| !self.execution.skipped_nodes.contains_key(key)) } pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { @@ -103,7 +116,7 @@ impl WorkerBuild { outcome, rows_affected, error_code, - skipped_because_of: None, + skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), }, )?; self.completed diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 472717ab03..52da11300d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -5,72 +5,68 @@ use std::time::Duration; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::types::PyString; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; +use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_orchestration::runs::models::RunOutcome; -use crate::bindings::_helpers::boundary::config_values::json_value_to_python; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; use crate::bindings::_helpers::build_command::command::run_build; +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; +use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, warehouse_scope}; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::native_host::host::{entry, write_console}; use crate::bindings::_helpers::recall::session::InvocationRecall; -use crate::bindings::_helpers::records::construct::new_record; -use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::fields::{field, replaced}; fn build_request<'py>( py: Python<'py>, discovered: Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, worker: &Bound<'py, NativeOrchestrationWorker>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; - for name in frozen.settings.keys() { - if !matches!( - name.as_str(), - "full_refresh" - | "fail_fast" - | "concurrency" - | "cli_vars" - | "run_tests" - | "run_audits" - | "load_sources" - | "reload_sources" - | "include_python" - | "allow_snapshot_full_refresh" - | "allow_snapshot_schema_change" - | "allow_table_type_downgrade" - | "allow_retention_decrease" - | "allow_missing_migration_origin" - | "max_microbatches" - | "test_timeout" - ) { - return Err(PyValueError::new_err(format!( - "unsupported worker build setting {name}" - ))); - } - } - let settings = PyDict::new(py); - for (name, value) in &frozen.settings { - settings.set_item(name, json_value_to_python(py, value)?)?; - } - settings.set_item("project_dir", arguments.value("project_dir")?)?; - settings.set_item("selected_target", frozen.target)?; - let mut selectors: Vec = Vec::new(); + let mut argv: Vec> = std::iter::once("build".to_owned()) + .chain(job_settings_arguments(&frozen.settings).map_err(PyValueError::new_err)?) + .map(|argument| PyString::new(py, &argument)) + .collect(); + argv.extend([ + PyString::new(py, "--project-dir"), + arguments.value("project_dir")?.str()?, + PyString::new(py, "--target"), + PyString::new(py, &frozen.target), + PyString::new(py, "--no-color"), + PyString::new(py, "--json"), + ]); for key in &frozen.selection { + if execution.skipped_nodes.contains_key(key) { + continue; + } let (_, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - selectors.push(name.to_owned()); + argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); } - settings.set_item("select", selectors)?; - settings.set_item("discovered_inputs", discovered)?; - settings.set_item("no_color", true)?; - settings.set_item("json_output", true)?; - new_record(py, RecordId::BuildCommandRequest, (), Some(&settings)) + let parsed = parse_cli_invocation(py, argv, false)?; + let args = field(&parsed, "args")?; + if args.is_none() { + return Err(PyValueError::new_err("invalid frozen build settings")); + } + let Dispatch::Run(route, None) = command_dispatch(&command_facts(&args)?) else { + return Err(PyValueError::new_err( + "frozen settings must dispatch a build", + )); + }; + let request = Invocation::new(&args)?.value(route.arguments[0].source)?; + Ok(( + replaced(&request, &[("discovered_inputs", discovered)])?, + warehouse_scope(&args)?, + )) } /// The heartbeat is native; only driver cancellation crosses the executor host boundary. @@ -130,13 +126,21 @@ pub(super) fn execute_worker<'py>( Some(Arc::clone(&worker)); } let result = (|| { - let request = build_request(py, discovered, arguments, &owner)?; + if !worker.has_work() { + write_console( + py, + "stderr", + "Run inputs verified. All selected nodes are blocked by upstream failures.\n", + )?; + return Ok(0); + } + let (request, scope) = build_request(py, discovered, arguments, &owner)?; write_console( py, "stderr", "Run inputs verified. Executing frozen selection...\n", )?; - run_cancellable(py, &worker, &request) + within(&scope, |_| run_cancellable(py, &worker, &request)) })(); let code = result.as_ref().copied().unwrap_or(1); let outcome = if recall.is_some() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 76ee58e258..d758b406a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use std::path::Path; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_host::entries::models::UserCodeHost; @@ -31,6 +31,7 @@ pub(crate) fn module_jobs<'py>( options.set_item("project_dir", project_dir)?; let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; + let group_class = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); @@ -39,7 +40,7 @@ pub(crate) fn module_jobs<'py>( if !seen.insert(value.as_ptr() as usize) { continue; } - if value.is_instance(&job_class)? { + if value.is_instance(&job_class)? || value.is_instance(&group_class)? { jobs.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { pending.extend(value.try_iter()?.collect::>>()?); @@ -48,17 +49,56 @@ pub(crate) fn module_jobs<'py>( Ok(jobs) } -pub(crate) fn validate_jobs( - py: Python<'_>, - declarations: &[Bound<'_, PyAny>], -) -> Result<(), PyErr> { +pub(crate) fn collect_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult>> { let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; - for declaration in declarations { - if !declaration.is_instance(&job)? { + let group = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let mut roots: HashSet = HashSet::new(); + for declaration in &declarations { + if !roots.insert(declaration.as_ptr() as usize) { + return Err(PyValueError::new_err(format!( + "duplicate job {}", + declaration.getattr("name")?.extract::()? + ))); + } + } + let mut pending = declarations; + let mut seen: HashSet = HashSet::new(); + let mut collected: Vec> = Vec::new(); + while let Some(declaration) = pending.pop() { + if !seen.insert(declaration.as_ptr() as usize) { + continue; + } + if declaration.is_instance(&group)? { + for member in declaration.getattr("jobs")?.try_iter()? { + let member = member?; + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err("group members must be Job objects")); + } + pending.push(member); + } + for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + for dependency in dependencies.try_iter()? { + if !dependency?.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + } + } + } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( - "snapshot declarations must be sqlbuild.orchestration.Job objects", + "snapshot declarations must be sqlbuild.orchestration.Job or JobGroup objects", )); } + collected.push(declaration); } - Ok(()) + Ok(collected) } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index e40b47cb19..6282151cb0 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod clickhouse_views; -pub(crate) mod postgres_views; pub(crate) mod relational_tables; pub(crate) mod relational_views; pub(crate) mod rendering; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs deleted file mode 100644 index ece90e302a..0000000000 --- a/crates/sqlbuild-recall/src/schema/_helpers/postgres_views.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! PostgreSQL expressions for the shared relational Recall view definitions. - -use crate::schema::_helpers::relational_views::relational_view_bodies; - -fn json_fields(mut body: String) -> String { - for (field, cast) in [ - ("audit_name", ""), - ("audit_definition_name", ""), - ("target_name", ""), - ("target_database", ""), - ("target_schema", ""), - ("attached_target_kind", ""), - ("attached_target_name", ""), - ("attached_column_name", ""), - ("severity", ""), - ("violation_count", "::BIGINT"), - ("measured_value", "::DOUBLE PRECISION"), - ("evaluation_mode", ""), - ("execution_error", ""), - ("evidence_count", "::BIGINT"), - ("evidence_truncated", "::BOOLEAN::INTEGER"), - ("sample_digest", ""), - ("error_message", ""), - ("signal_number", "::BIGINT"), - ("cost_status", ""), - ] { - body = body.replace( - &format!("json_extract(payload, '$.{field}')"), - &format!("(payload ->> '{field}'){cast}"), - ); - } - for (field, cast) in [ - ("name", ""), - ("job_name", ""), - ("run_id", ""), - ("retry_number", "::BIGINT"), - ] { - body = body.replace( - &format!("json_extract(context.external_context, '$.integration.{field}')"), - &format!("(context.external_context #>> '{{integration,{field}}}'){cast}"), - ); - } - body.replace( - "json_valid(context.external_context)", - "context.external_context IS NOT NULL", - ) - .replace( - "json_array_length(context.selection)", - "jsonb_array_length(context.selection)", - ) -} - -/// Keep display bytes until readers decode them: PostgreSQL text rejects NUL and malformed UTF-8. -fn postgres_body(body: String) -> String { - let mut body = json_fields(body) - .replace("strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", "CURRENT_TIMESTAMP") - .replace("MAX(artifact.digest IS NOT NULL)", "MAX(CAST(artifact.digest IS NOT NULL AS INTEGER))") - .replace("artifact.digest IS NOT NULL AS available", "CAST(artifact.digest IS NOT NULL AS INTEGER) AS available") - .replace("MAX(\n COALESCE(received.max_sequence", "GREATEST(\n COALESCE(received.max_sequence") - .replace("MAX(COALESCE(received.queue_drops", "GREATEST(COALESCE(received.queue_drops") - .replace("COALESCE(context.recall_filter_active, 0)", "COALESCE(CAST(context.recall_filter_active AS INTEGER), 0)") - .replace("COALESCE(delivery.flush_complete, 0)", "COALESCE(CAST(delivery.flush_complete AS INTEGER), 0)") - .replace("CAST(data AS TEXT)", "data") - .replace("CAST(artifact.content AS TEXT)", "artifact.content") - .replace(")\nGROUP BY", ") AS source_rows\nGROUP BY") - .replace(")\n GROUP BY", ") AS source_rows\n GROUP BY") - .replace(")\nWHERE position", ") AS source_rows\nWHERE position") - .replace("ROUND(\n (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3\n )", - "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)"); - for expression in [ - "byte_count", - "CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END", - "CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END", - "dropped_before", - "COALESCE(omitted_sequences, 1)", - "COALESCE(omitted_bytes, 0)", - ] { - body = body.replace( - &format!("SUM({expression})"), - &format!("CAST(SUM({expression}) AS BIGINT)"), - ); - } - body -} - -pub(crate) fn postgres_view_bodies() -> Vec<(&'static str, String)> { - relational_view_bodies(true) - .into_iter() - .map(|(name, body)| (name, postgres_body(body))) - .collect() -} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index c2bffe4987..321bdb28a8 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,11 +1,15 @@ -//! Shared relational view definitions, with explicit dialect expressions for binary transcripts. +//! Shared relational views emit dialect differences at each expression's definition. +use crate::schema::_helpers::rendering::{json_field, relational_cast}; use crate::schema::constants::OUTPUT_TAIL_RECORDS; -/// The current time as stored timestamps spell it. -const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; +const EVENTS: &str = "SELECT * FROM recall_events"; +const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; -const STATEMENTS: &str = "WITH facts AS ( +fn statements(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + format!("WITH facts AS ( SELECT * FROM recall_events WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' @@ -41,7 +45,7 @@ terminals AS ( ), captures AS ( SELECT firsts.statement_id, ( - SELECT MAX(artifact.digest IS NOT NULL) + SELECT MAX({available}) FROM recall_artifact_refs AS ref LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest WHERE ref.invocation_id = firsts.invocation_id @@ -102,9 +106,13 @@ LEFT JOIN captures ON captures.statement_id = firsts.statement_id LEFT JOIN resources ON resources.invocation_id = firsts.invocation_id AND resources.resource_id = firsts.resource_id -WHERE firsts.position = 1"; +WHERE firsts.position = 1") +} -const AUDIT_RESULTS: &str = "SELECT +fn audit_results(postgres: bool) -> String { + let field = |path, cast| json_field(postgres, ("payload", path), cast); + format!( + "SELECT invocation_id, invocation_started_at, event_id, @@ -112,40 +120,63 @@ const AUDIT_RESULTS: &str = "SELECT run_id, resource_id, resource_attempt_id, - json_extract(payload, '$.audit_name') AS audit_name, - json_extract(payload, '$.audit_definition_name') AS audit_definition_name, - CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE - COALESCE(json_extract(payload, '$.target_database') || '.', '') - || COALESCE(json_extract(payload, '$.target_schema') || '.', '') - || json_extract(payload, '$.target_name') + {audit_name} AS audit_name, + {audit_definition_name} AS audit_definition_name, + CASE WHEN {target_name} IS NULL THEN NULL ELSE + COALESCE({target_database} || '.', '') + || COALESCE({target_schema} || '.', '') + || {target_name} END AS target_relation, - json_extract(payload, '$.attached_target_kind') AS attached_target_kind, - json_extract(payload, '$.attached_target_name') AS attached_target_name, - json_extract(payload, '$.attached_column_name') AS \"column\", + {attached_target_kind} AS attached_target_kind, + {attached_target_name} AS attached_target_name, + {attached_column_name} AS \"column\", outcome, - json_extract(payload, '$.severity') AS severity, - json_extract(payload, '$.violation_count') AS violation_count, - json_extract(payload, '$.measured_value') AS measured_value, - json_extract(payload, '$.evaluation_mode') AS evaluation_mode, - json_extract(payload, '$.execution_error') AS execution_error, + {severity} AS severity, + {violation_count} AS violation_count, + {measured_value} AS measured_value, + {evaluation_mode} AS evaluation_mode, + {execution_error} AS execution_error, sql_digest, - json_extract(payload, '$.evidence_count') AS evidence_count, - json_extract(payload, '$.evidence_truncated') AS evidence_truncated, - json_extract(payload, '$.sample_digest') AS sample_digest + {evidence_count} AS evidence_count, + {evidence_truncated} AS evidence_truncated, + {sample_digest} AS sample_digest FROM recall_events -WHERE event_type = 'audit_completed'"; +WHERE event_type = 'audit_completed'", + audit_name = field("audit_name", ""), + audit_definition_name = field("audit_definition_name", ""), + target_name = field("target_name", ""), + target_database = field("target_database", ""), + target_schema = field("target_schema", ""), + attached_target_kind = field("attached_target_kind", ""), + attached_target_name = field("attached_target_name", ""), + attached_column_name = field("attached_column_name", ""), + severity = field("severity", ""), + violation_count = field("violation_count", "::BIGINT"), + measured_value = field("measured_value", "::DOUBLE PRECISION"), + evaluation_mode = field("evaluation_mode", ""), + execution_error = field("execution_error", ""), + evidence_count = field("evidence_count", "::BIGINT"), + evidence_truncated = field("evidence_truncated", "::BOOLEAN::INTEGER"), + sample_digest = field("sample_digest", "") + ) +} -const OUTPUT_INVOCATIONS: &str = "WITH received AS ( +fn output_invocations(postgres: bool) -> String { + let sum = |expression| relational_cast(postgres, (&format!("SUM({expression})"), "BIGINT")); + let maximum = if postgres { "GREATEST" } else { "MAX" }; + let filter = relational_cast(postgres, ("context.recall_filter_active", "INTEGER")); + let flush = relational_cast(postgres, ("delivery.flush_complete", "INTEGER")); + format!("WITH received AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, COUNT(*) AS chunks, - SUM(byte_count) AS bytes, - SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, - SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + {bytes} AS bytes, + {stdout_bytes} AS stdout_bytes, + {stderr_bytes} AS stderr_bytes, MAX(sequence) AS max_sequence, - SUM(dropped_before) AS queue_drops, - SUM(COALESCE(omitted_sequences, 1)) AS spans, - SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + {drops} AS queue_drops, + {spans} AS spans, + {truncated} AS truncated_bytes FROM recall_output GROUP BY invocation_id ), @@ -166,15 +197,15 @@ counted AS ( COALESCE(received.stderr_bytes, 0) AS stderr_bytes, COALESCE(received.truncated_bytes, 0) AS truncated_bytes, received.max_sequence, - MAX( + {maximum}( COALESCE(received.max_sequence + 1 - received.spans, 0), COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) ) AS missing_sequences, - MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + {maximum}(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, COALESCE(delivery.filtered, 0) AS filtered, COALESCE(delivery.failed, 0) AS failed, - COALESCE(context.recall_filter_active, 0) AS filter_active, - COALESCE(delivery.flush_complete, 0) AS flush_complete, + COALESCE({filter}, 0) AS filter_active, + COALESCE({flush}, 0) AS flush_complete, delivery.invocation_id IS NOT NULL AS delivery_reported, COALESCE(delivery.delivered, 0) AS delivered FROM ids @@ -210,13 +241,21 @@ SELECT AND chunks >= delivered THEN 1 ELSE 0 END AS complete -FROM counted"; - -const EVENTS: &str = "SELECT * FROM recall_events"; - -const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +FROM counted", + bytes = sum("byte_count"), stdout_bytes = sum("CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END"), + stderr_bytes = sum("CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END"), drops = sum("dropped_before"), + spans = sum("COALESCE(omitted_sequences, 1)"), truncated = sum("COALESCE(omitted_bytes, 0)")) +} -const ARTIFACTS: &str = "SELECT +fn artifacts(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + let text = if postgres { + "artifact.content" + } else { + "CAST(artifact.content AS TEXT)" + }; + format!( + "SELECT refs.invocation_id, refs.invocation_started_at, refs.digest, @@ -227,15 +266,15 @@ const ARTIFACTS: &str = "SELECT artifact.kind, artifact.media_type, artifact.size, - artifact.digest IS NOT NULL AS available, + {available} AS available, CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' - THEN CAST(artifact.content AS TEXT) END AS text, + THEN {text} END AS text, artifact.content FROM recall_artifact_refs AS refs -LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; - -const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest" + ) +} fn output_transcripts(postgres: bool) -> String { let columns: Vec = [ @@ -261,7 +300,27 @@ fn output_transcripts(postgres: bool) -> String { ) } -fn invocations() -> String { +fn invocations(postgres: bool) -> String { + let now = if postgres { + "CURRENT_TIMESTAMP" + } else { + "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')" + }; + let selection_count = if postgres { + "jsonb_array_length(context.selection)" + } else { + "json_array_length(context.selection)" + }; + let valid_context = if postgres { + "context.external_context IS NOT NULL" + } else { + "json_valid(context.external_context)" + }; + let elapsed = if postgres { + "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)" + } else { + "ROUND((unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3)" + }; format!( "WITH ids AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at @@ -270,7 +329,7 @@ fn invocations() -> String { UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') - ) + ) AS source_rows GROUP BY invocation_id ), started AS ( @@ -281,9 +340,9 @@ started AS ( ), terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, - json_extract(payload, '$.error_message') AS error_message, - json_extract(payload, '$.signal_number') AS signal_number, - json_extract(payload, '$.cost_status') AS cost_status, + {error_message} AS error_message, + {signal_number} AS signal_number, + {cost_status} AS cost_status, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -308,7 +367,7 @@ projected AS ( AND terminals.signal_number IS NOT NULL THEN 'cancelled' WHEN terminals.event_type = 'invocation_failed' THEN 'failed' WHEN beats.stale_at IS NULL THEN 'unknown' - WHEN beats.stale_at >= {NOW} THEN 'running' + WHEN beats.stale_at >= {now} THEN 'running' ELSE 'abandoned' END AS status, terminals.event_type AS terminal_event, @@ -328,7 +387,7 @@ projected AS ( context.adapter, context.project_name, context.selection, - COALESCE(context.selection_count, json_array_length(context.selection)) + COALESCE(context.selection_count, {selection_count}) AS selection_count, context.selection_truncated, context.selector_files, @@ -339,17 +398,17 @@ projected AS ( context.host_token, context.pid, context.external_context, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.name') END + CASE WHEN {valid_context} + THEN {integration} END AS integration, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.job_name') END + CASE WHEN {valid_context} + THEN {integration_job} END AS integration_job, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.run_id') END + CASE WHEN {valid_context} + THEN {integration_run_id} END AS integration_run_id, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.retry_number') END + CASE WHEN {valid_context} + THEN {integration_retry} END AS integration_retry, context.recall_filter_active, beats.last_heartbeat_at @@ -371,9 +430,7 @@ SELECT END AS ended_at, CASE WHEN terminal_event IS NOT NULL THEN terminal_duration_ms - WHEN status = 'abandoned' THEN ROUND( - (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 - ) + WHEN status = 'abandoned' THEN {elapsed} END AS duration_ms, CASE WHEN terminal_event IS NOT NULL THEN 0 @@ -409,18 +466,46 @@ SELECT integration_retry, recall_filter_active, last_heartbeat_at -FROM projected" +FROM projected", + error_message = json_field(postgres, ("payload", "error_message"), ""), + signal_number = json_field(postgres, ("payload", "signal_number"), "::BIGINT"), + cost_status = json_field(postgres, ("payload", "cost_status"), ""), + integration = json_field( + postgres, + ("context.external_context", "integration.name"), + "" + ), + integration_job = json_field( + postgres, + ("context.external_context", "integration.job_name"), + "" + ), + integration_run_id = json_field( + postgres, + ("context.external_context", "integration.run_id"), + "" + ), + integration_retry = json_field( + postgres, + ("context.external_context", "integration.retry_number"), + "::BIGINT" + ) ) } -fn output_tail() -> String { +fn output_tail(postgres: bool) -> String { + let text = if postgres { + "data" + } else { + "CAST(data AS TEXT)" + }; format!( "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, - CAST(data AS TEXT) AS text + {text} AS text FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position FROM recall_output -) +) AS source_rows WHERE position <= {OUTPUT_TAIL_RECORDS}" ) } @@ -428,15 +513,15 @@ WHERE position <= {OUTPUT_TAIL_RECORDS}" /// Each supported view's name and relational body, in creation order. pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { vec![ - ("invocations", invocations()), - ("statements", STATEMENTS.to_owned()), - ("audit_results", AUDIT_RESULTS.to_owned()), - ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), - ("output_tail", output_tail()), + ("invocations", invocations(postgres)), + ("statements", statements(postgres)), + ("audit_results", audit_results(postgres)), + ("output_invocations", output_invocations(postgres)), + ("output_tail", output_tail(postgres)), ("output_transcripts", output_transcripts(postgres)), ("events", EVENTS.to_owned()), ("diagnostics", DIAGNOSTICS.to_owned()), - ("artifacts", ARTIFACTS.to_owned()), + ("artifacts", artifacts(postgres)), ("conflicts", CONFLICTS.to_owned()), ] } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs index d7f3eb635b..f7355f0442 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -68,6 +68,22 @@ pub(crate) fn clickhouse_ttl(spec: &TableSpec, options: &DdlOptions) -> String { format!("{} + INTERVAL {days} DAY", spec.expiry_column) } +pub(crate) fn json_field(postgres: bool, (column, path): (&str, &str), cast: &str) -> String { + if postgres { + format!("({column} #>> '{{{}}}'){cast}", path.replace('.', ",")) + } else { + format!("json_extract({column}, '$.{path}')") + } +} + +pub(crate) fn relational_cast(postgres: bool, (expression, kind): (&str, &str)) -> String { + if postgres { + format!("CAST({expression} AS {kind})") + } else { + expression.into() + } +} + /// The engine clause, replicated with one shard and every replica on a cluster. pub(crate) fn clickhouse_engine(spec: &TableSpec, options: &DdlOptions) -> String { let version: &str = spec.clickhouse_version.unwrap_or_default(); diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs index 826dd58587..1645d1ab19 100644 --- a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -2,7 +2,7 @@ use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; -use crate::schema::_helpers::postgres_views::postgres_view_bodies; +use crate::schema::_helpers::relational_views::relational_view_bodies; use crate::schema::main::view_statements::view_statements; use crate::schema::models::{DdlOptions, Dialect}; @@ -10,7 +10,7 @@ use crate::schema::models::{DdlOptions, Dialect}; #[must_use] pub fn control_view_statements(postgres: bool) -> Vec { if postgres { - postgres_view_bodies() + relational_view_bodies(true) .into_iter() .map(|(name, body)| { format!( diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 94bb6c3ace..d305deef89 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,8 +1,9 @@ """Public authoring and snapshot compilation API for orchestration declarations.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models import Job, Schedule +from sqlbuild.orchestration_declarations.models import Job, JobGroup, Schedule from sqlbuild.orchestration_declarations.models import Job as job +from sqlbuild.orchestration_declarations.models import JobGroup as job_group from sqlbuild.orchestration_declarations.models import Schedule as schedule -__all__ = ("Job", "Schedule", "compile_snapshot", "job", "schedule") +__all__ = ("Job", "JobGroup", "Schedule", "compile_snapshot", "job", "job_group", "schedule") diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index 3cba43a00c..cc10a5eb4a 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,7 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import Job +from sqlbuild.orchestration_declarations.models import Job, JobGroup def compile_snapshot( @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job] | None = None, + declarations: Sequence[Job | JobGroup] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index fa8ff9efc1..e113ce4aa4 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass, field from typing import Literal @@ -35,3 +36,17 @@ class Job: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, eq=False) +class JobGroup: + """Jobs ordered by compiled lineage and optional explicit dependencies.""" + + name: str + description: str = field(kw_only=True) + jobs: Sequence[Job] = field(kw_only=True) + after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) + overlap: Literal["coalesce", "queue", "allow"] = field(default="coalesce", kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 53f479aa3c..293ae24452 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -252,6 +252,7 @@ class SnapshotDiscoveryTestCase: files: dict[str, str] expected_jobs: tuple[str, ...] declarations: tuple[()] | None = None + expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 7b78e9f190..135c64aaf6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -102,9 +102,19 @@ class WorkerExecutionTestCase: files: dict[str, str] = field(default_factory=dict) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" + concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" +@dataclass(frozen=True) +class GroupExecutionTestCase(WorkerExecutionTestCase): + description: str + on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = "skip_dependents" + expected_child_statuses: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_totals: tuple[str, str | None] = ("skipped", "model:broken_orders") + expected_marts_started: bool = True + + @dataclass(frozen=True) class WorkerRestartTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a91b471edf..1b98f71fb1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -322,7 +322,14 @@ def scheduled_worker( "server_url": api.url, "max_workers": 1, } | deployment.configuration - _ = configuration.write_text(json.dumps({deployment.kind: [launcher]})) + _ = configuration.write_text( + json.dumps( + { + deployment.kind: [launcher], + "concurrency_limits": test_case.concurrency_limits, + } + ) + ) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) @@ -380,10 +387,14 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - return run -def running_statement(worker: ScheduledWorker) -> dict[str, Any]: +def running_statement(worker: ScheduledWorker, *, job_key: str | None = None) -> dict[str, Any]: """The worker has started the real long query, not merely entered its Python interpreter.""" active: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, pending=frozenset({"queued", "launched"}) + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued", "launched"}), + job_key=job_key, ) assert active["status"] == "running", worker.log.read_text() deadline: float = time.monotonic() + 15 @@ -398,9 +409,11 @@ def running_statement(worker: ScheduledWorker) -> dict[str, Any]: return active -def cancel_running_statement(worker: ScheduledWorker) -> None: +def cancel_running_statement( + worker: ScheduledWorker, *, job_key: str | None = None, run_id: str | None = None +) -> None: """Wait for the real long-running query before exercising the administrative cancel command.""" - active: dict[str, Any] = running_statement(worker) + active: dict[str, Any] = running_statement(worker, job_key=job_key) cancelled: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -408,7 +421,7 @@ def cancel_running_statement(worker: ScheduledWorker) -> None: "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", "cancel-run", - active["run_id"], + run_id or active["run_id"], "--event-id", str(uuid4()), "--reason", @@ -428,6 +441,7 @@ def wait_for_run( log: Path, *, pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), + job_key: str | None = None, ) -> dict[str, Any]: """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" request: Request = Request( @@ -440,7 +454,10 @@ def wait_for_run( assert process.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() with urlopen(request, timeout=2) as response: - current = next(iter(json.load(response)), current) + runs: list[dict[str, Any]] = json.load(response) + by_job: dict[str | None, dict[str, Any]] = {run["job_key"]: run for run in runs} + by_job[None] = next(iter(runs), current) + current = by_job.get(job_key, current) time.sleep(0.02) return current diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 8658227d69..a95849fa86 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -5,11 +5,13 @@ import signal import sqlite3 import time +from dataclasses import replace from pathlib import Path from typing import Any from urllib.request import Request, urlopen from uuid import uuid4 +import duckdb import pytest from sqlbuild._native import ( @@ -26,6 +28,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, + GroupExecutionTestCase, SnapshotWorkerTestCase, WorkerExecutionTestCase, ) @@ -35,11 +38,220 @@ scheduled_worker, snapshot_api, wait_for_log, + wait_for_run, worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +@pytest.mark.parametrize( + "test_case", + [ + GroupExecutionTestCase( + "group execution preserves independent downstream work", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT amount FROM orders", + "partial", + [(11,)], + expected_child_statuses={"staging": ("partial",), "marts": ("partial",)}, + ), + GroupExecutionTestCase( + "successful jobs honor compiled boundaries", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + ), + GroupExecutionTestCase( + "whole downstream job is blocked without building an empty selection", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "partial", + [(0,)], + on_failure="skip_downstream_jobs", + expected_child_statuses={"staging": ("partial",), "marts": ("failed",)}, + ), + GroupExecutionTestCase( + "job failure stops the group", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + on_failure="stop_group", + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + ), + GroupExecutionTestCase( + "critical node stops the group under the default policy", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + files={ + "models/broken_orders.sql": ( + "MODEL (description 'Critical orders.', materialized table, critical true);\n" + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_failure_policy_hold( + test_case: GroupExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + with duckdb.connect(str(project_dir / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE broken_orders AS SELECT 99 AS amount") + test_case = replace( + test_case, + files={ + "models/orders.sql": ( + "MODEL (description 'Independent orders.', materialized table);\n" + 'SELECT amount FROM __ref("stg_orders")\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\nSELECT 11 AS amount\n" + ), + "models/broken_orders.sql": ( + "MODEL (description 'Source values.', materialized table);\n" + test_case.model_sql + ), + "models/totals.sql": ( + "MODEL (description 'Dependent totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + 'staging = job("staging", description="Stage orders.", ' + f'select="stg_orders broken_orders", on_failure="{test_case.on_failure}")\n' + 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' + 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' + "jobs=[marts, staging], " + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert parent["status"] == test_case.expected_status, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: dict[str, Any] = {run["job_key"]: run for run in json.load(response)} + assert set(runs) == {"annual_orders", *test_case.expected_child_statuses} + assert all( + runs[name]["status"] in statuses + for name, statuses in test_case.expected_child_statuses.items() + ) + assert (runs["marts"]["started_at"] is not None) == test_case.expected_marts_started + assert runs["staging"]["ended_at"] <= ( + runs["marts"]["started_at"] or runs["staging"]["ended_at"] + ) + with sqlite3.connect(worker.control_store) as control: + assert ( + control.execute( + "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" + ).fetchall() + == [(runs["annual_orders"]["run_id"],) * 2] * 2 + ) + assert ( + control.execute( + "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=? AND node_key='model:totals'", + (runs["marts"]["run_id"],), + ).fetchone() + == test_case.expected_totals + ) + assert control.execute( + "SELECT status FROM sqb_run_nodes WHERE run_id=? AND node_key='model:broken_orders'", + (runs["staging"]["run_id"],), + ).fetchone() == ({"success": "success"}.get(test_case.expected_status, "failed"),) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + with duckdb.connect(str(project_dir / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="opposing parent and independent child limits", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM inventory ORDER BY amount", + expected_status="success", + expected_rows=[(11,), (23,)], + concurrency_limits={"orders_limit": 1, "inventory_limit": 1}, + files={ + "models/inventory.sql": ( + "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders_job', description='Build orders.', select='orders', " + "concurrency_groups=('inventory_limit',))\n" + "inventory = job('inventory_job', description='Build inventory.', select='inventory', " + "concurrency_groups=('orders_limit',))\n" + "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " + "concurrency_groups=('orders_limit',), " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_inventory = job_group('annual_inventory', description='Inventory group.', " + "jobs=[inventory], concurrency_groups=('inventory_limit',), " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + store_setup=( + "CREATE TABLE test_slot_claims (run_id TEXT, group_key TEXT, occupied INTEGER);\n" + "CREATE TRIGGER test_observe_slots AFTER INSERT ON sqb_concurrency_slots BEGIN " + "INSERT INTO test_slot_claims SELECT NEW.run_id, NEW.group_key, count(*) " + "FROM sqb_concurrency_slots WHERE group_key=NEW.group_key; END;\n" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_opposing_group_limits_when_scheduled_then_both_real_workers_finish_without_hoisting_limits( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + orders: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + inventory: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_inventory" + ) + assert orders["status"] == inventory["status"] == "success", worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " + "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" + ).fetchall() == [ + ("annual_inventory", "inventory_limit", 1), + ("annual_orders", "orders_limit", 1), + ("inventory_job", "orders_limit", 1), + ("orders_job", "inventory_limit", 1), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE status='success'" + ).fetchone() == (4,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ @@ -167,6 +379,27 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "success", [(23,)], ), + WorkerExecutionTestCase( + "authored variables and reload use ordinary build argument conversion", + "SELECT @@batch AS amount", + "SELECT amount FROM orders", + "success", + [(23,)], + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[vars]\nbatch = "3"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.', " + "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), WorkerExecutionTestCase( "warehouse error retains failed node evidence", "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", @@ -235,6 +468,68 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "parent cancellation interrupts its active child and cancels its waiting child", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('orders','totals')", + "cancelled", + [(0,)], + files={ + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders', description='Long query.', select='orders')\n" + "totals = job('totals', description='Waiting query.', select='totals')\n" + "annual_orders = job_group('annual_orders', description='Cancellable group.', " + "jobs=[totals, orders], schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_before_parent_completion( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued"}), + job_key="annual_orders", + ) + assert parent["status"] == "running", worker.log.read_text() + cancel_running_statement(worker, job_key="orders", run_id=parent["run_id"]) + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key, status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "cancelled"), + ("orders", "cancelled"), + ("totals", "cancelled"), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", + (parent["run_id"], completed["ended_at"]), + ).fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 03b75e763a..f274c11f7e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -554,6 +554,29 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, expected_jobs=("daily", "staging"), ), + SnapshotDiscoveryTestCase( + description="group-only references retain lineage and explicit waits", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "def factory():\n" + " staging = job('staging', description='Stage orders.', select='stg_orders')\n" + " marts = job('marts', description='Build orders.', select='orders')\n" + " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" + " return job_group('daily', description='Ordered jobs.', " + "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "daily = factory()\n" + ), + }, + expected_jobs=("inventory", "marts", "staging"), + expected_groups={ + "daily": { + "inventory": ["staging"], + "marts": ["inventory", "staging"], + "staging": [], + } + }, + ), SnapshotDiscoveryTestCase( description="explicit empty declarations do not execute project files", files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, @@ -570,7 +593,16 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "MODEL (", "MODEL (tags ['edge_case'],", 1 ), }, - expected_jobs=("edge_snapshot_orders",), + expected_jobs=( + "edge_snapshot_inventory", + "edge_snapshot_orders", + "edge_snapshot_products", + ), + expected_groups={ + "edge_snapshot_group": {"edge_snapshot_orders": []}, + "edge_inventory_group": {"edge_snapshot_inventory": []}, + "edge_products_group": {"edge_snapshot_products": []}, + }, ), SnapshotDiscoveryTestCase( description="a project without declarations has no job document", @@ -598,6 +630,10 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ).to_json() ) assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs + assert { + name: definition["jobs"] + for name, definition in publication["documents"].get("groups", {}).items() + } == test_case.expected_groups for definition in publication["documents"].get("jobs", {}).values(): assert set(definition["selection"]) <= publication["nodes"].keys() @@ -605,6 +641,44 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects @pytest.mark.parametrize( "test_case", [ + SnapshotDiscoveryRejectionTestCase( + description="group members are jobs, not nested groups", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/nested_groups.py" + ).read_text(), + }, + expected_error=TypeError, + expected_message="group members must be Job objects", + ), + SnapshotDiscoveryRejectionTestCase( + description="explicit group cycles cannot publish", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Cyclic jobs.', " + "jobs=[marts, staging], after={staging: [marts]})\n" + ), + }, + expected_error=ValueError, + expected_message="group dependencies contain a cycle", + ), + SnapshotDiscoveryRejectionTestCase( + description="group waits cannot refer to nonmember jobs", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Unknown upstream.', " + "jobs=[marts], after={marts: [staging]})\n" + ), + }, + expected_error=ValueError, + expected_message="group needs known jobs and dependencies within its membership", + ), SnapshotDiscoveryRejectionTestCase( description="distinct objects cannot share a job name", files={ From 1bb63c7193fa35ed5175be93b177272115f7b362 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:13:01 +0000 Subject: [PATCH 037/138] fix: preserve failure-only reruns and transitive job ordering Keep successful and cancelled nodes out of failure-only reruns, retain waits through omitted completed jobs, and prove the complete local and Docker worker paths. The backend remains WIP. SQLite/PostgreSQL API, run and Recall regressions pass; the publication suite passes 42 cases. The exact static target still reports six known unused-public-entry findings. --- .../orchestration/rerun_from_failure.sql | 8 ++ .../sqlbuild-cli/src/argv/command_table.json | 1 + .../argv/tests/test_parse_cli_arguments.rs | 29 ++++ .../src/runs/_helpers/groups/selection.rs | 27 ++-- .../src/runs/_helpers/requests/job.rs | 57 ++++++++ .../src/runs/main/queue_run.rs | 55 +------ .../tests/runs/test_reruns.rs | 134 +++++++++++------- .../tests/runs/test_types.rs | 12 +- .../compiled_project/publication/helpers.py | 5 +- .../publication/test_rerun.py | 124 ++++++++++++++++ 10 files changed, 336 insertions(+), 116 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql new file mode 100644 index 0000000000..8304e49843 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql @@ -0,0 +1,8 @@ +MODEL (description 'Retry a failed cast after warehouse input changes.', materialized table); +SELECT CASE + WHEN EXISTS ( + SELECT 1 FROM information_schema.tables WHERE table_name = 'retry_ready' + ) THEN 17 + ELSE CAST(value AS INTEGER) +END AS amount +FROM (VALUES ('invalid')) AS source(value) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ed0917d366..fea155edee 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -493,6 +493,7 @@ "options": ["--from-failure"], "dest": "admin_from_failure", "nargs": 0, + "const": true, "default": false, "help": "retry exactly the failed and skipped nodes, excluding successful nodes", "group": 1 diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index ab5d022216..62ba60f4db 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -49,6 +49,35 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "contract_command", expected_value: Value::Str("diff".to_owned()), }, + ParsedValueTestCase { + description: "an explicit failed-node rerun never becomes a full rerun", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry failed orders", + "--from-failure", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(true), + }, + ParsedValueTestCase { + description: "a full rerun is the default without failure narrowing", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry all orders", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(false), + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs index a42f48821c..4f2c66310c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -28,7 +28,7 @@ pub(crate) async fn child_ready( .job_key .as_ref() .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; - let dependencies = &group.definitions.groups[parent_name].jobs[name]; + let dependencies = upstream_jobs(&group.definitions.groups[parent_name].jobs, name); let ready = group.parent.state.status == RunStatus::Running && group.parent.state.cancel_requested_at.is_none() && !group.stopped() @@ -37,7 +37,7 @@ pub(crate) async fn child_ready( .request_json .job_key .as_ref() - .is_none_or(|name| !dependencies.contains(name)) + .is_none_or(|name| !dependencies.contains(name.as_str())) || child.state.status.is_terminal() }); Ok((transaction, ready)) @@ -62,13 +62,7 @@ pub(crate) async fn skipped_nodes( .as_ref() .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; let edges = &group.definitions.groups[parent_name].jobs; - let mut ancestors: BTreeSet<&str> = BTreeSet::new(); - let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); - while let Some(name) = pending.pop() { - if ancestors.insert(name) { - pending.extend(edges[name].iter().map(String::as_str)); - } - } + let ancestors = upstream_jobs(edges, name); let (transaction, rows) = query(transaction, "SELECT n.node_key, n.status, n.skipped_because_of FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ WHERE r.parent_run_id=$1 AND r.run_id<>$2 AND n.status IN ('failed','skipped','cancelled')", @@ -137,3 +131,18 @@ pub(crate) async fn skipped_nodes( } Ok((transaction, skipped)) } + +/// Completed jobs omitted from a retry do not sever explicit ordering between retained jobs. +fn upstream_jobs<'a>( + edges: &'a BTreeMap>, + name: &str, +) -> BTreeSet<&'a str> { + let mut ancestors: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); + while let Some(name) = pending.pop() { + if ancestors.insert(name) { + pending.extend(edges[name].iter().map(String::as_str)); + } + } + ancestors +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index edd8116dd6..edbfae8f23 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -1,6 +1,11 @@ //! Freeze execution inputs from one published job or group in the caller's write transaction. +use std::collections::BTreeSet; + +use uuid::Uuid; + use crate::credentials::models::AuditContext; +use crate::declarations::main::frozen_definitions::frozen_definitions; use crate::declarations::models::JobDefinition; use crate::runs::main::job_write_set::job_write_set; use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; @@ -63,3 +68,55 @@ pub(crate) async fn frozen_job_request( }, )) } + +pub(crate) async fn frozen_group_children( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; + let group = request + .job_key + .as_ref() + .and_then(|name| definitions.groups.get(name)) + .ok_or_else(|| ControlError::Invalid("group is absent from the pinned snapshot".into()))?; + let selection_matches = if request.origin == RunOrigin::Rerun { + request.selection.is_subset(&group.job.selection) + } else { + group.job.selection == request.selection + }; + if !selection_matches || !request.write_set.is_empty() { + return Err(ControlError::Invalid( + "group scope differs from its pinned membership".into(), + )); + } + let (transaction, writers) = + job_write_set(transaction, request.snapshot_id, &request.selection).await?; + let mut children: Vec = Vec::new(); + for name in group.jobs.keys() { + let job = &definitions.jobs[name]; + let selection: BTreeSet<_> = job + .selection + .intersection(&request.selection) + .cloned() + .collect(); + if selection.is_empty() { + continue; + } + children.push(RunRequest { + run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), + kind: RunKind::Job, + job_key: Some(name.clone()), + parent_run_id: Some(request.run_id), + rerun_of_run_id: request + .rerun_of_run_id + .map(|id| Uuid::new_v5(&id, name.as_bytes())), + write_set: selection.intersection(&writers).cloned().collect(), + selection, + settings: job.settings.clone(), + concurrency_groups: job.concurrency_groups.clone(), + overlap: job.overlap, + ..request.clone() + }); + } + Ok((transaction, children)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs index b8cb2e9b89..fb4d86c590 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -4,10 +4,9 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; -use crate::declarations::main::frozen_definitions::frozen_definitions; use crate::runs::_helpers::lifecycle::events::append_run_event; -use crate::runs::main::job_write_set::job_write_set; -use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::models::{RunKind, RunRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -18,53 +17,9 @@ pub(crate) async fn queue_run( ) -> Result { let mut transaction = queue_one(transaction, request).await?; if request.kind == RunKind::Group { - let definitions; - (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; - let group = request - .job_key - .as_ref() - .and_then(|name| definitions.groups.get(name)) - .ok_or_else(|| { - ControlError::Invalid("group is absent from the pinned snapshot".into()) - })?; - let selection_matches = if request.origin == RunOrigin::Rerun { - request.selection.is_subset(&group.job.selection) - } else { - group.job.selection == request.selection - }; - if !selection_matches || !request.write_set.is_empty() { - return Err(ControlError::Invalid( - "group scope differs from its pinned membership".into(), - )); - } - let writers; - (transaction, writers) = - job_write_set(transaction, request.snapshot_id, &request.selection).await?; - for name in group.jobs.keys() { - let job = &definitions.jobs[name]; - let selection: std::collections::BTreeSet<_> = job - .selection - .intersection(&request.selection) - .cloned() - .collect(); - if selection.is_empty() { - continue; - } - let child = RunRequest { - run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), - kind: RunKind::Job, - job_key: Some(name.clone()), - parent_run_id: Some(request.run_id), - rerun_of_run_id: request - .rerun_of_run_id - .map(|id| Uuid::new_v5(&id, name.as_bytes())), - write_set: selection.intersection(&writers).cloned().collect(), - selection, - settings: job.settings.clone(), - concurrency_groups: job.concurrency_groups.clone(), - overlap: job.overlap, - ..request.clone() - }; + let children; + (transaction, children) = frozen_group_children(transaction, request).await?; + for child in children { transaction = queue_one(transaction, &child).await?; } } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index a17b54cee9..0729862d1f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -23,7 +23,7 @@ use crate::helpers::snapshot; use crate::location_helpers::create_location; use crate::store_helpers::{close_test_store, open_test_store, run}; use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::test_types::{RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault}; #[test] fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() @@ -57,11 +57,6 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ip_address: None, }; let mut input = snapshot(location); - input - .nodes - .get_mut("inventory") - .expect("inventory") - .definition["deps"] = json!(["orders"]); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, @@ -69,7 +64,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ })); input.documents.insert("groups".into(), json!({"daily_orders": { "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"], "completed": []} + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); @@ -88,7 +83,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .await .expect("request group"); let retry = RerunRequest { - run_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), source_run_id: source_id, from_failure: true, reason: "Retry failed orders".into(), @@ -103,16 +98,30 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("elect") .expect("leader"); let limits = BTreeMap::from([ - ("warehouse".into(), std::num::NonZeroU32::new(1).unwrap()), + ( + "warehouse".into(), + std::num::NonZeroU32::new(1).expect("positive capacity"), + ), ( "inventory_limit".into(), - std::num::NonZeroU32::new(1).unwrap(), + std::num::NonZeroU32::new(1).expect("positive capacity"), ), ]); let scope = (location, "development"); advance_groups(store, &lease, (&limits, scope)) .await .expect("admit source"); + let node_outcomes = BTreeMap::from([ + ("products", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ("customers", NodeOutcome::Cancelled), + ("inventory", NodeOutcome::Skipped), + ]); + let job_outcomes = BTreeMap::from([ + ("staging", RunOutcome::Failed), + ("completed", RunOutcome::Success), + ("marts", RunOutcome::Failed), + ]); for _ in 0..3 { let claim = claim_next_run(store, &lease, &limits, Some(scope)) .await @@ -120,12 +129,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("ready child"); let mut changes = vec![RunChange::Started]; for node in &claim.request.selection { - let outcome = match node.as_str() { - "products" => NodeOutcome::Success, - "orders" => NodeOutcome::Failed, - "customers" => NodeOutcome::Cancelled, - _ => NodeOutcome::Skipped, - }; + let outcome = node_outcomes[node.as_str()]; changes.push(RunChange::NodeFinished { node_key: node.clone(), outcome, @@ -136,11 +140,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ }); } changes.push(RunChange::Finished { - outcome: if claim.request.selection.contains("products") { - RunOutcome::Success - } else { - RunOutcome::Failed - }, + outcome: job_outcomes[claim.request.job_key.as_deref().expect("source job")], reason: "synthetic source outcome".into(), exit_code: Some(1), }); @@ -168,16 +168,16 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ assert_eq!( list_runs(store, scope, 20) .await - .unwrap() + .expect("list completed source") .iter() .find(|run| run.run_id == source_id) - .unwrap() + .expect("source exists") .status, RunStatus::Partial ); input.generation = 2; input.image_id = Some("sha256:synthetic-orders-image-2".into()); - input.documents.get_mut("jobs").unwrap()["staging"]["settings"] = + input.documents.get_mut("jobs").expect("job declarations")["staging"]["settings"] = json!({"vars": {"batch": 91}}); publish_snapshot(store, &input).await.expect("redeploy"); let fault = TransactionFault { @@ -192,7 +192,10 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ assert!(request_rerun(store, &context, &retry).await.is_err()); remove_fault(store, &fault).await; assert_eq!( - list_runs(store, scope, 20).await.unwrap().len(), + list_runs(store, scope, 20) + .await + .expect("list rolled-back runs") + .len(), 4, "audit failure rolls back parent and all children" ); @@ -211,41 +214,67 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ..context.clone() }; assert!(request_rerun(store, &other, &retry).await.is_err()); - for request in [ - retry.clone(), - RerunRequest { - run_id: Uuid::now_v7(), - from_failure: false, - ..retry.clone() + let settings = BTreeMap::from([ + ( + "staging", + BTreeMap::from([("vars".into(), json!({"batch": 23}))]), + ), + ("completed", BTreeMap::new()), + ("marts", BTreeMap::new()), + ]); + for execution in [ + RerunExecutionTestCase { + description: "failure retry omits successful and cancelled nodes", + request: retry.clone(), + expected_applied: false, + expected_selection: ["orders", "inventory"].map(str::to_owned).into(), + expected_jobs: &["staging", "marts"], }, - ] { - if !request.from_failure { - assert!(request_rerun(store, &context, &request).await.unwrap()); - } - let expected: BTreeSet = if request.from_failure { - ["orders", "inventory"].map(str::to_owned).into() - } else { - ["orders", "inventory", "customers", "products"] + RerunExecutionTestCase { + description: "full retry retains all original nodes", + request: RerunRequest { + run_id: Uuid::now_v7(), + from_failure: false, + ..retry.clone() + }, + expected_applied: true, + expected_selection: ["orders", "inventory", "customers", "products"] .map(str::to_owned) - .into() - }; - let runs = list_runs(store, scope, 20).await.unwrap(); + .into(), + expected_jobs: &["staging", "completed", "marts"], + }, + ] { + let request = execution.request; + assert_eq!( + request_rerun(store, &context, &request) + .await + .expect("publish rerun"), + execution.expected_applied, + "{}", + execution.description + ); + let runs = list_runs(store, scope, 20).await.expect("list reruns"); assert_eq!( runs.iter() .find(|run| run.run_id == request.run_id) - .unwrap() + .expect("rerun exists") .selection_json, - expected + execution.expected_selection ); advance_groups(store, &lease, (&limits, scope)) .await .expect("admit rerun"); - let mut selected = BTreeSet::new(); - for _ in 0..if request.from_failure { 2 } else { 3 } { + let mut selected: BTreeSet = BTreeSet::new(); + for expected_job in execution.expected_jobs { let claim = claim_next_run(store, &lease, &limits, Some(scope)) .await .expect("claim rerun") .expect("ready retry child"); + assert_eq!( + claim.request.job_key.as_deref(), + Some(*expected_job), + "a completed intermediate job must not erase its ancestor's wait boundary" + ); assert_eq!(claim.request.snapshot_id, published.snapshot_id); assert_eq!( claim.request.image_id, @@ -253,26 +282,21 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ); assert_eq!(claim.request.origin, RunOrigin::Rerun); assert_eq!(claim.request.priority, 7); - if claim.request.job_key.as_deref() == Some("staging") { - assert_eq!( - claim.request.settings, - BTreeMap::from([("vars".into(), json!({"batch": 23}))]) - ); - } + assert_eq!(claim.request.settings, settings[expected_job]); selected.extend(claim.request.selection.iter().cloned()); succeed(store, &claim).await; } - assert_eq!(selected, expected); + assert_eq!(selected, execution.expected_selection); advance_groups(store, &lease, (&limits, scope)) .await .expect("finish rerun"); assert_eq!( list_runs(store, scope, 20) .await - .unwrap() + .expect("list finished reruns") .iter() .find(|run| run.run_id == request.run_id) - .unwrap() + .expect("finished rerun exists") .status, test_case.expected_status, "{}", diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 9ee50b5f8c..0646241008 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -1,6 +1,8 @@ //! Expected queue outcomes for each real control-store backend. -use sqlbuild_orchestration::runs::models::RunStatus; +use std::collections::BTreeSet; + +use sqlbuild_orchestration::runs::models::{RerunRequest, RunStatus}; pub(crate) use crate::fault_types::TransactionFault; use crate::store_test_types::Backend; @@ -50,6 +52,14 @@ pub(crate) struct RunLifecycleTestCase { pub(crate) expected_status: RunStatus, } +pub(crate) struct RerunExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) request: RerunRequest, + pub(crate) expected_applied: bool, + pub(crate) expected_selection: BTreeSet, + pub(crate) expected_jobs: &'static [&'static str], +} + pub(crate) struct RunMonitorTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 1b98f71fb1..f0717f1000 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -442,6 +442,7 @@ def wait_for_run( *, pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), job_key: str | None = None, + run_id: str | None = None, ) -> dict[str, Any]: """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" request: Request = Request( @@ -457,7 +458,9 @@ def wait_for_run( runs: list[dict[str, Any]] = json.load(response) by_job: dict[str | None, dict[str, Any]] = {run["job_key"]: run for run in runs} by_job[None] = next(iter(runs), current) - current = by_job.get(job_key, current) + by_id: dict[str | None, dict[str, Any]] = {run["run_id"]: run for run in runs} + by_id[None] = by_job.get(job_key, current) + current = by_id.get(run_id, current) time.sleep(0.02) return current diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py new file mode 100644 index 0000000000..f7edf07364 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -0,0 +1,124 @@ +"""Pinned group retries rebuild failed and skipped nodes without repeating successful writes.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="local group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + ), + WorkerExecutionTestCase( + description="Docker group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_are_not_rewritten( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files={ + "models/broken_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql" + ).read_text(), + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "models/inventory.sql": ( + "MODEL (description 'Independent inventory.', materialized table);\n" + "SELECT 29 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" + "marts = job('marts', description='Build totals.', select='totals')\n" + "inventory = job('inventory', description='Build inventory.', select='inventory')\n" + "annual_orders = job_group('annual_orders', description='Retry boundaries.', " + "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 1 1 *', " + "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + source: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert source["status"] == "partial", worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE retry_ready AS SELECT 1 AS id") + warehouse.execute("UPDATE orders SET amount=99") + warehouse.execute("UPDATE inventory SET amount=71") + run_id: str = str(uuid4()) + command: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + source["run_id"], + "--run-id", + run_id, + "--from-failure", + "--reason", + "Retry failed and skipped nodes after fixing warehouse input.", + ] + for expected_applied in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + command, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected_applied} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + assert completed["snapshot_id"] == source["snapshot_id"] + assert completed["selection_json"] == ["model:broken_orders", "model:totals"] + assert completed["node_counts_json"] == {"success": 2} + with sqlite3.connect(worker.control_store) as control: + children: list[tuple[str, str]] = control.execute( + "SELECT job_key, selection_json FROM sqb_runs WHERE parent_run_id=? ORDER BY job_key", + (run_id,), + ).fetchall() + assert [(name, json.loads(selection)) for name, selection in children] == [ + ("marts", ["model:totals"]), + ("staging", ["model:broken_orders"]), + ] + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='run.rerun_requested'" + ).fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + assert warehouse.sql("SELECT amount FROM inventory").fetchall() == [(71,)] From 6df49478e489c8f515564a2cbf9e32bdb71d4c9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:13:14 +0000 Subject: [PATCH 038/138] test: complete scheduled-function DuckShop coverage Exercise the inherited seed and scalar function together, attach the required not-null audit and assert concrete SQL unit-test results. The failing DuckShop compile checks now pass without relaxing any Rules or changing the core benchmark checksum. --- .../edge_cases/rpt_edge_scheduled_function.sql | 11 ++++++++++- .../test_rpt_edge_scheduled_function.sql | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql index 5e33820e53..f86364144c 100644 --- a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql @@ -2,6 +2,15 @@ MODEL ( description 'A scheduled model whose implicit scalar-function writer must be claimed.', tags [edge_case], materialized table, + columns ( + quantity (type INTEGER, nullable false, audits [not_null]), + ), ); -SELECT __udf("edge_next_quantity")(CAST(23 AS INTEGER)) AS quantity +WITH final AS ( + SELECT __udf("edge_next_quantity")(q.quantity) AS quantity + FROM __seed("edge_quantity_steps") AS q +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..8db1011dad --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql @@ -0,0 +1,15 @@ +TEST (name "rpt_edge_scheduled_function__advances_the_inherited_function_quantity"); + +WITH __seed__edge_quantity_steps AS ( + SELECT v.quantity + FROM VALUES + (-2), + (23) AS v(quantity) +), + +__expected__rpt_edge_scheduled_function AS ( + SELECT v.quantity + FROM VALUES + (-1), + (24) AS v(quantity) +) From 2836325a965b5f6e16893f376c35484f10a3ffae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:58:25 +0000 Subject: [PATCH 039/138] feat: recover Linux local workers (WIP) Retain inherited claim locks through scheduler death and verify physical absence before releasing claims. Acquire ownership before fenced launch publication so a delayed stale scheduler cannot spawn after recovery. WIP: local restart recovery is Linux-only. The backend remains incomplete and the static gate is red on six existing test/API-boundary findings; no checks are bypassed or relaxed. --- Cargo.lock | 11 ++ Cargo.toml | 1 + .../orchestration/local_worker_recovery.sql | 3 + crates/sqlbuild-cli/Cargo.toml | 3 + .../src/workers/_helpers/child.rs | 6 + .../src/workers/_helpers/local.rs | 128 ++++++++++++ .../sqlbuild-cli/src/workers/_helpers/mod.rs | 2 + .../src/workers/_helpers/recovery.rs | 28 ++- .../src/workers/_helpers/supervisor.rs | 43 ++-- crates/sqlbuild-cli/src/workers/constants.rs | 4 + crates/sqlbuild-cli/src/workers/models.rs | 11 ++ .../publication/test_local_recovery.py | 184 ++++++++++++++++++ 12 files changed, 402 insertions(+), 22 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql create mode 100644 crates/sqlbuild-cli/src/workers/_helpers/local.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py diff --git a/Cargo.lock b/Cargo.lock index dbef530346..d3c2ae6c05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -348,6 +348,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" +[[package]] +name = "command-fds" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b60b5124979fccd9addd89d8b97a1d6eebb4950694520c75ddd722535ea443f" +dependencies = [ + "nix 0.31.3", + "thiserror 2.0.21", +] + [[package]] name = "command-group" version = "5.0.1" @@ -2493,6 +2503,7 @@ version = "0.186.2" dependencies = [ "blake3", "chrono", + "command-fds", "ctrlc", "getrandom 0.4.3", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 61061e23b9..fd65e8af7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,6 +99,7 @@ syn-v3-macro-features = { package = "syn", version = "3", default-features = fal thiserror = "2" # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it. ctrlc = "3" +command-fds = "0.3" quote = "1" rayon = "1" # Static reading of project Python source; the parser fensu uses. diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql new file mode 100644 index 0000000000..fe752e60d2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql @@ -0,0 +1,3 @@ +-- A long-running statement for scheduler crash, adoption and cancellation checks. +-- The supervising test cancels this query; it is not part of benchmark measurements. +SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value) diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 5568005174..cac9dd214d 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -36,6 +36,9 @@ uuid.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +command-fds.workspace = true + [target.'cfg(windows)'.dependencies] ctrlc.workspace = true same-file.workspace = true diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs index a6ca868c82..75d44e336f 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/child.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -22,6 +22,8 @@ impl WorkerChild { Self::Local(child) => Ok(child .try_wait()? .map(|exit| WorkerExit::Code(exit_code(exit)))), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.try_wait(), Self::Docker(child) => child.try_wait().await, } } @@ -41,6 +43,8 @@ impl WorkerChild { pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { match self { Self::Local(child) => terminate(child), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM), Self::Docker(child) => child.signal("SIGTERM").await, } } @@ -51,6 +55,8 @@ impl WorkerChild { child.start_kill()?; Ok(()) } + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::KILL), Self::Docker(child) => child.signal("SIGKILL").await, } } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/local.rs new file mode 100644 index 0000000000..82a1c4fba2 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/local.rs @@ -0,0 +1,128 @@ +//! Linux dev workers inherit a claim lock before exec; recovery never guesses from a stale PID. + +use std::fs::OpenOptions; +use std::os::unix::fs::MetadataExt; + +use command_fds::{CommandFdExt, FdMapping}; +use rustix::fs::{FlockOperation, flock}; +use rustix::process::{Pid, PidfdFlags, Signal, pidfd_open, pidfd_send_signal}; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::process::Command; + +use crate::workers::constants::LOCAL_CLAIM_FD; +use crate::workers::models::{LocalLauncher, LocalWorker, WorkerExit}; + +impl LocalWorker { + pub(in crate::workers) fn open( + location: &LocalLauncher, + claim: &RunClaim, + ) -> Result { + let directory = location.project_dir.join(".sqlbuild/workers"); + std::fs::create_dir_all(&directory)?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(directory.join(format!("{}.lock", claim.claim_id)))?; + Ok(Self { + lock, + run_id: claim.request.run_id, + stopped: false, + }) + } + + /// Keep the acquired lock until terminal publication, so a delayed spawn cannot start afterward. + pub(in crate::workers) fn try_wait(&mut self) -> Result, ControlError> { + if !self.stopped { + match flock(&self.lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => self.stopped = true, + Err(rustix::io::Errno::WOULDBLOCK) => return Ok(None), + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(Some(WorkerExit::Absent)) + } + + /// Only a kernel handle whose current descriptor and invocation match this claim can be signalled. + pub(in crate::workers) fn signal(&self, signal: Signal) -> Result<(), ControlError> { + let expected = self.lock.metadata()?; + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let raw = match name.parse::() { + Ok(raw) => raw, + Err(_) => continue, + }; + let Some(pid) = Pid::from_raw(raw) else { + continue; + }; + let descriptor = entry.path().join(format!("fd/{LOCAL_CLAIM_FD}")); + if !same_file(&descriptor, &expected) { + continue; + } + let handle = match pidfd_open(pid, PidfdFlags::empty()) { + Ok(handle) => handle, + Err(rustix::io::Errno::SRCH) => continue, + Err(error) => return Err(std::io::Error::from(error).into()), + }; + if !same_file(&descriptor, &expected) || !self.owns_invocation(&entry.path())? { + continue; + } + match pidfd_send_signal(handle, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(()) + } + + fn owns_invocation(&self, process: &std::path::Path) -> Result { + let environment = match std::fs::read(process.join("environ")) { + Ok(environment) => environment, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + let prefix = format!("{INVOCATION_CONTEXT_ENV}="); + for variable in environment.split(|byte| *byte == 0) { + if let Some(context) = variable.strip_prefix(prefix.as_bytes()) { + let context: serde_json::Value = + serde_json::from_slice(context).map_err(|error| { + ControlError::Invalid(format!( + "invalid local worker invocation context: {error}" + )) + })?; + return Ok(context["run_id"].as_str() == Some(&self.run_id.to_string())); + } + } + Ok(false) + } +} + +fn same_file(path: &std::path::Path, expected: &std::fs::Metadata) -> bool { + std::fs::metadata(path) + .is_ok_and(|actual| actual.dev() == expected.dev() && actual.ino() == expected.ino()) +} + +/// The child inherits the lock even before Python starts or a worker handshake can succeed. +pub(in crate::workers) fn inherit_claim( + command: &mut Command, + location: &LocalLauncher, + claim: &RunClaim, +) -> Result<(), ControlError> { + let owner = LocalWorker::open(location, claim)?; + flock(&owner.lock, FlockOperation::NonBlockingLockExclusive).map_err(std::io::Error::from)?; + command + .as_std_mut() + .fd_mappings(vec![FdMapping { + parent_fd: owner.lock.into(), + child_fd: LOCAL_CLAIM_FD, + }]) + .map_err(|error| ControlError::Invalid(format!("local claim descriptor: {error}")))?; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs index b182827d2d..a0ead33080 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -2,6 +2,8 @@ mod child; pub(super) mod configuration; mod docker; mod invocation; +#[cfg(target_os = "linux")] +mod local; mod recovery; mod shutdown; mod supervisor; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs index 1704f88f6b..c54d87b613 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -9,7 +9,7 @@ use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; -use crate::workers::models::{ClaimedWorker, DockerWorker, WorkerChild, Workers}; +use crate::workers::models::{ClaimedWorker, DockerWorker, Launcher, WorkerChild, Workers}; impl Workers { pub(in crate::workers) async fn recover( @@ -25,12 +25,17 @@ impl Workers { .iter() .map(|worker| worker.claim.claim_id) .collect(); - for location in &self.configuration.docker { + for location in self.configuration.locations() { + #[cfg(not(target_os = "linux"))] + if matches!(location, Launcher::Local(_)) { + continue; + } + let scope = location.scope(); let mut after = None; loop { renew_lease(store, lease).await?; let claims = - list_run_claims(store, (location.location_id, &location.target), after).await?; + list_run_claims(store, (scope.location_id, scope.target), after).await?; let Some(last) = claims.last() else { break; }; @@ -39,10 +44,19 @@ impl Workers { if !known.insert(claim.claim_id) { continue; } - let child = match DockerWorker::prepare(location, &claim) { - Ok(child) => Some(WorkerChild::Docker(child)), - Err(ControlError::Invalid(_)) => None, - Err(error) => return Err(error), + let child = match location { + #[cfg(target_os = "linux")] + Launcher::Local(location) => Some(WorkerChild::RecoveredLocal( + crate::workers::models::LocalWorker::open(location, &claim)?, + )), + #[cfg(not(target_os = "linux"))] + Launcher::Local(_) => continue, + Launcher::Docker(location) => match DockerWorker::prepare(location, &claim) + { + Ok(child) => Some(WorkerChild::Docker(child)), + Err(ControlError::Invalid(_)) => None, + Err(error) => return Err(error), + }, }; drop(writeln!( std::io::stderr().lock(), diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index b46fe96bab..b9aa97e091 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -219,6 +219,15 @@ impl Workers { Utc::now() + TimeDelta::days(7), ) .await?; + let local_command = match location { + Launcher::Local(location) => Some(prepare_spawn( + location, + claim, + &credential.bearer, + &self.control_variable, + )?), + Launcher::Docker(_) => None, + }; publish( store, lease, @@ -226,16 +235,17 @@ impl Workers { ("local-launched", RunChange::Launched { container_id }), ) .await?; - let launched = match location { - Launcher::Local(location) => { - spawn(location, claim, &credential.bearer, &self.control_variable).map( - |child| { - worker.child = Some(WorkerChild::Local(child)); - }, - ) - } - Launcher::Docker(location) => { - if let Some(WorkerChild::Docker(child)) = &worker.child { + let launched = match local_command { + Some(mut command) => command + .spawn() + .map(|child| { + worker.child = Some(WorkerChild::Local(child)); + }) + .map_err(ControlError::from), + None => { + if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = + (location, &worker.child) + { child .launch(location, claim, &credential.bearer, &self.control_variable) .await?; @@ -275,13 +285,14 @@ impl Workers { } } -fn spawn( +fn prepare_spawn( location: &LocalLauncher, claim: &RunClaim, bearer: &str, control_variable: &str, -) -> Result { - Ok(Command::new(&location.executable) +) -> Result { + let mut command = Command::new(&location.executable); + command .args(arguments(&Launcher::Local(location).scope(), claim)) .arg(&location.project_dir) .current_dir(&location.project_dir) @@ -294,8 +305,10 @@ fn spawn( .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn()?) + .kill_on_drop(true); + #[cfg(target_os = "linux")] + crate::workers::_helpers::local::inherit_claim(&mut command, location, claim)?; + Ok(command) } pub(in crate::workers) async fn publish( diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs index 881c48bea1..06a889dbfb 100644 --- a/crates/sqlbuild-cli/src/workers/constants.rs +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -1,3 +1,7 @@ //! Immutable Docker image identities use the full SHA-256 hexadecimal digest. pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; + +/// Keep inherited claim ownership away from standard IO and the runtime's usual low descriptors. +#[cfg(target_os = "linux")] +pub(super) const LOCAL_CLAIM_FD: i32 = 198; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 3cd00b537f..415247a7f7 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -13,9 +13,20 @@ use uuid::Uuid; #[derive(Debug)] pub(super) enum WorkerChild { Local(Child), + #[cfg(target_os = "linux")] + RecoveredLocal(LocalWorker), Docker(DockerWorker), } +/// The inherited lock proves physical ownership without relying on a remembered process ID. +#[cfg(target_os = "linux")] +#[derive(Debug)] +pub(super) struct LocalWorker { + pub(super) lock: std::fs::File, + pub(super) run_id: Uuid, + pub(super) stopped: bool, +} + #[derive(Clone, Copy, Debug)] pub(super) enum WorkerExit { Code(i64), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py new file mode 100644 index 0000000000..938a1b34b3 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -0,0 +1,184 @@ +"""A replacement Linux dev scheduler retains physical ownership of surviving local workers.""" + +import os +import signal +import sqlite3 +import sys +import time +from contextlib import suppress +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_interpreter, + wait_for_log, + wait_for_run, + worker_result, +) + +fcntl: ModuleType = pytest.importorskip("fcntl") + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "scheduler SIGKILL does not orphan a running local warehouse statement", + Path( + "benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql" + ).read_text(), + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_and_cancels_without_relaunch( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + with sqlite3.connect(worker.control_store) as control: + claim_id: str = control.execute( + "SELECT worker_instance FROM sqb_run_liveness WHERE run_id=?", (active["run_id"],) + ).fetchone()[0] + lock: Path = worker.project / ".sqlbuild/workers" / f"{claim_id}.lock" + with lock.open("rb") as ownership: + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + deadline: float = time.monotonic() + 10 + while True: + try: + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pre-handshake child survives restart and failed terminal publication retains its claim", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + store_setup=( + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status='failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_wait_for_physical_exit_and_commit( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pid: int = wait_for_interpreter(worker.process, worker.project / "worker.pid", worker.log) + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued"}) + ) + assert active["status"] == "launched" + handle: int = os.pidfd_open(pid) + try: + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + signal.pidfd_send_signal(handle, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + "launched", + ) + signal.pidfd_send_signal(handle, signal.SIGKILL) + wait_for_log(process, log, "Retrying worker shutdown publication") + assert process.poll() is None, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + _ = control.execute("DROP TRIGGER reject_terminal") + assert process.wait(timeout=10) != 0, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + test_case.expected_status, + ) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 0, + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (None,) + assert not (worker.project / "warehouse.duckdb").exists() + finally: + with suppress(ProcessLookupError): + signal.pidfd_send_signal(handle, signal.SIGKILL) + os.close(handle) From ac937265a714820d7e4ecb2082cca3345a282e6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:26:09 +0000 Subject: [PATCH 040/138] refactor: keep scheduler internals owner-local Remove test-only production convenience exports and retain the real SQLite/PostgreSQL contracts in owner-local test harnesses. Exercise CLI worker lease setup through the public scheduler loop. Shared fixtures are declared once under their owners. --- .../tests/native_commands/worker/helpers.rs | 57 +++++++++++++++++-- .../native_commands/worker/test_types.rs | 7 +++ crates/sqlbuild-orchestration/src/api/mod.rs | 4 ++ .../src/credentials/mod.rs | 4 ++ .../src/platform/mod.rs | 4 ++ .../src/runs/main/mod.rs | 1 - .../src/runs/main/request_run.rs | 16 ------ crates/sqlbuild-orchestration/src/runs/mod.rs | 4 ++ .../scheduler/_helpers/drain_due_triggers.rs | 2 +- .../{acquire_lease.rs => _acquire_lease.rs} | 2 +- ...aluate_trigger.rs => _evaluate_trigger.rs} | 2 +- ...scheduler_cycle.rs => _scheduler_cycle.rs} | 4 +- .../src/scheduler/main/mod.rs | 6 +- .../src/scheduler/main/run_scheduler.rs | 2 +- .../src/scheduler/mod.rs | 2 +- .../src/scheduler/tests.rs | 2 + .../src/snapshots/mod.rs | 4 ++ .../src/store/main/mod.rs | 2 - .../src/store/main/publish_recall.rs | 25 -------- .../src/store/main/read_recall_events.rs | 28 --------- .../sqlbuild-orchestration/src/store/mod.rs | 4 ++ crates/sqlbuild-orchestration/tests/api.rs | 28 --------- .../tests/api/helpers.rs | 7 +-- .../tests/api/output/helpers.rs | 10 ++-- .../tests/api/recall/helpers.rs | 12 ++-- .../tests/api/test_contract.rs | 48 ++++++++-------- .../tests/api/test_contracts.rs | 22 +++++++ .../tests/api/test_location_health.rs | 28 +++++---- .../tests/api/test_location_work.rs | 38 ++++++------- .../tests/api/test_reads.rs | 49 ++++++++-------- .../tests/api/test_types.rs | 7 +-- .../tests/api/test_worker.rs | 46 +++++++-------- .../tests/api/work/helpers.rs | 2 +- .../tests/credentials.rs | 20 ------- .../tests/credentials/helpers.rs | 2 +- .../tests/credentials/locations/helpers.rs | 8 +-- .../tests/credentials/test_contract.rs | 36 ++++++------ .../tests/credentials/test_contracts.rs | 10 ++++ .../tests/credentials/test_types.rs | 4 +- .../sqlbuild-orchestration/tests/platform.rs | 14 ----- .../tests/platform/helpers.rs | 12 ++-- .../tests/platform/test_contracts.rs | 8 +++ .../tests/platform/test_types.rs | 4 +- .../tests/platform/test_watchdog.rs | 22 +++---- crates/sqlbuild-orchestration/tests/runs.rs | 42 -------------- .../tests/runs/claims/helpers.rs | 6 +- .../tests/runs/helpers.rs | 23 ++++++-- .../tests/runs/monitor/helpers.rs | 14 ++--- .../tests/runs/termination/helpers.rs | 10 ++-- .../tests/runs/test_claims.rs | 40 +++++++------ .../tests/runs/test_coalesce.rs | 25 ++++---- .../tests/runs/test_contracts.rs | 28 +++++++++ .../tests/runs/test_faults.rs | 34 +++++------ .../tests/runs/test_group_admission.rs | 28 ++++----- .../tests/runs/test_groups.rs | 31 +++++----- .../tests/runs/test_lifecycle.rs | 34 +++++------ .../tests/runs/test_monitor.rs | 47 ++++++++------- .../tests/runs/test_requests.rs | 30 +++++----- .../tests/runs/test_reruns.rs | 30 +++++----- .../tests/runs/test_types.rs | 6 +- .../sqlbuild-orchestration/tests/scheduler.rs | 22 ------- .../tests/scheduler/helpers.rs | 20 ++++--- .../tests/scheduler/test_contract.rs | 8 +-- .../tests/scheduler/test_contracts.rs | 10 ++++ .../tests/scheduler/test_types.rs | 2 +- .../tests/scheduler/ticks/helpers.rs | 38 ++++++------- .../tests/scheduler/ticks/test_contract.rs | 8 +-- .../tests/scheduler/ticks/test_loop.rs | 37 ++++++------ .../tests/scheduler/ticks/test_triggers.rs | 34 +++++------ .../tests/scheduler/ticks/test_types.rs | 18 +++--- .../sqlbuild-orchestration/tests/snapshots.rs | 14 ----- .../tests/snapshots/helpers.rs | 24 ++++---- .../tests/snapshots/test_contract.rs | 8 +-- .../tests/snapshots/test_contracts.rs | 8 +++ .../tests/snapshots/test_types.rs | 2 +- crates/sqlbuild-orchestration/tests/store.rs | 16 ------ .../tests/store/faults/helpers.rs | 4 +- .../tests/store/helpers.rs | 6 +- .../tests/store/leases/helpers.rs | 2 +- .../tests/store/recall/helpers.rs | 51 +++++++++++++++-- .../tests/store/recall/test_contract.rs | 10 ++-- .../tests/store/recall/test_types.rs | 2 +- .../tests/store/recall/test_views.rs | 12 ++-- .../tests/store/replicas/helpers.rs | 4 +- .../tests/store/test_contracts.rs | 22 +++++++ .../tests/store/test_types.rs | 2 +- 86 files changed, 714 insertions(+), 717 deletions(-) delete mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_run.rs rename crates/sqlbuild-orchestration/src/scheduler/main/{acquire_lease.rs => _acquire_lease.rs} (97%) rename crates/sqlbuild-orchestration/src/scheduler/main/{evaluate_trigger.rs => _evaluate_trigger.rs} (98%) rename crates/sqlbuild-orchestration/src/scheduler/main/{scheduler_cycle.rs => _scheduler_cycle.rs} (96%) delete mode 100644 crates/sqlbuild-orchestration/src/store/main/publish_recall.rs delete mode 100644 crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs delete mode 100644 crates/sqlbuild-orchestration/tests/api.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/credentials.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/platform.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/runs.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/scheduler.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/snapshots.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs delete mode 100644 crates/sqlbuild-orchestration/tests/store.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/test_contracts.rs diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index d9635b34d3..5a5a7fc4a0 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -11,10 +11,14 @@ use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::{SchedulerLease, SchedulerState}; +use sqlbuild_orchestration::scheduler::types::RunLauncher; use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_orchestration::worker::models::WorkerSession; @@ -23,7 +27,33 @@ use uuid::Uuid; use crate::admin_helpers::administer; use crate::helpers::isolated_binary; use crate::test_types::TestControl; -use crate::worker::test_types::WorkerControl; +use crate::worker::test_types::{LeaseCapture, WorkerControl}; + +impl RunLauncher for LeaseCapture { + async fn poll( + &mut self, + _store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + if let Some(lease) = lease { + self.lease + .take() + .expect("first scheduler lease") + .send(lease.clone()) + .expect("lease receiver"); + self.shutdown.send(true).expect("scheduler shutdown"); + } + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &ControlStore, + _lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + Ok(()) + } +} pub(crate) async fn start_claimed_worker( control: &TestControl, @@ -120,10 +150,25 @@ pub(crate) async fn claim_worker_run( [&run], |row| row.get(0)).expect("trusted manual request audit"); assert_eq!(count, 1); drop(connection); - let lease = acquire_lease(&store, Uuid::now_v7()) - .await - .expect("acquire lease") - .expect("leader"); + let (lease_sender, lease_receiver) = tokio::sync::oneshot::channel(); + let (shutdown, signal) = tokio::sync::watch::channel(false); + let (_changes, changes) = tokio::sync::watch::channel(()); + tokio::time::timeout( + Duration::from_secs(5), + run_scheduler( + &store, + SchedulerState::new(Uuid::now_v7()), + (signal, changes), + LeaseCapture { + lease: Some(lease_sender), + shutdown, + }, + ), + ) + .await + .expect("scheduler acquires leadership promptly") + .expect("scheduler shutdown"); + let lease = lease_receiver.await.expect("public scheduler leadership"); let claim = claim_next_run( &store, &lease, diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs index 54d23d6116..81ecbc260f 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -1,5 +1,12 @@ use crate::test_types::TestControl; +pub(crate) struct LeaseCapture { + pub(crate) lease: Option< + tokio::sync::oneshot::Sender, + >, + pub(crate) shutdown: tokio::sync::watch::Sender, +} + pub(crate) struct WorkerHeartbeatTestCase { pub(crate) description: &'static str, pub(crate) interruption: fn(WorkerControl, &mut TestControl), diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index 951756c642..cb776dbca6 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -5,3 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/api/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/credentials/mod.rs b/crates/sqlbuild-orchestration/src/credentials/mod.rs index a3ce0d1f39..acde203166 100644 --- a/crates/sqlbuild-orchestration/src/credentials/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/mod.rs @@ -4,3 +4,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/credentials/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs index 5359000d16..32d92d3cc2 100644 --- a/crates/sqlbuild-orchestration/src/platform/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -1,3 +1,7 @@ pub(crate) mod _helpers; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/platform/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index a69509d123..6359a19779 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -13,5 +13,4 @@ pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_job_run; pub mod request_rerun; -pub mod request_run; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_run.rs deleted file mode 100644 index f3041afbbc..0000000000 --- a/crates/sqlbuild-orchestration/src/runs/main/request_run.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Queue a frozen, already authorized request without recreating work on transport retries. - -use crate::runs::_helpers::requests::publication::publish_request; -use crate::runs::models::RunRequest; -use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; - -/// The API must compute and authorize this selection; callers cannot supply their own write scope. -pub async fn request_run(store: &ControlStore, request: &RunRequest) -> Result { - let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, applied) = publish_request(transaction, request).await?; - commit(transaction).await?; - Ok(applied) -} diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs index f147865646..529a59766a 100644 --- a/crates/sqlbuild-orchestration/src/runs/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -2,3 +2,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/runs/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index ab6a639022..f2edb08a87 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::scheduler::main::evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs index 1c1bf2ec6a..20b4286e1b 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/acquire_lease.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs @@ -12,7 +12,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; /// An existing live lease is never stolen, including by a duplicate caller with the same holder. -pub async fn acquire_lease( +pub(crate) async fn acquire_lease( store: &ControlStore, holder: Uuid, ) -> Result, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs index 7d065991ee..2646e237ee 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -17,7 +17,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; /// A removed or paused trigger is inert; a lost fence rolls back every write, including Recall. -pub async fn evaluate_trigger( +pub(crate) async fn evaluate_trigger( store: &ControlStore, lease: &SchedulerLease, trigger_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs rename to crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs index 8db0fea9b5..1d94e6944a 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -8,7 +8,7 @@ use crate::runs::constants::MONITOR_BATCH_SIZE; use crate::runs::main::monitor_runs::monitor_runs; use crate::scheduler::_helpers::drain_due_triggers::drain_due_triggers; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; -use crate::scheduler::main::acquire_lease::acquire_lease; +use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; @@ -17,7 +17,7 @@ use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, TransactionMode}; -pub async fn scheduler_cycle( +pub(crate) async fn scheduler_cycle( store: &ControlStore, mut state: SchedulerState, ) -> Result<(SchedulerState, SchedulerCycle), ControlError> { diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs index 60225c3875..f91c3d1180 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -1,12 +1,12 @@ +pub(in crate::scheduler) mod _acquire_lease; +pub(in crate::scheduler) mod _evaluate_trigger; pub(in crate::scheduler) mod _next_cron_time; pub(in crate::scheduler) mod _plan_ticks; pub(in crate::scheduler) mod _previous_cron_time; -pub mod acquire_lease; +pub(in crate::scheduler) mod _scheduler_cycle; pub(crate) mod check_lease; -pub mod evaluate_trigger; pub mod list_ticks; pub(crate) mod next_schedule_time; pub mod parse_cron; pub mod renew_lease; pub mod run_scheduler; -pub mod scheduler_cycle; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs index b4fd01d5e5..71954a8f71 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -2,7 +2,7 @@ use tokio::sync::watch; -use crate::scheduler::main::scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; use crate::scheduler::types::RunLauncher; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs index cf840ff555..f5b66b8aca 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -7,4 +7,4 @@ pub mod models; pub mod types; #[cfg(test)] -mod tests; +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs index e0731c81e8..927f476ce9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -1,3 +1,5 @@ +#[path = "../../tests/scheduler/test_contracts.rs"] +pub(crate) mod contracts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_backlog.rs"] diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs index 5359000d16..04ae56c307 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -1,3 +1,7 @@ pub(crate) mod _helpers; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/snapshots/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index 715e39afaf..d8119ad010 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -6,8 +6,6 @@ pub(crate) mod decode_row; pub mod list_recall_invocations; pub mod migrate_store; pub mod open_store; -pub mod publish_recall; pub(crate) mod query; -pub mod read_recall_events; pub mod read_recall_view; pub(crate) mod refresh_clock; diff --git a/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs b/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs deleted file mode 100644 index bbe77db9a8..0000000000 --- a/crates/sqlbuild-orchestration/src/store/main/publish_recall.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Worker publication is atomic; scheduler writes use the same append inside their tick transaction. - -use sqlbuild_recall::records::models::RecallRow; - -use crate::store::errors::ControlError; -use crate::store::main::append_recall::append_recall; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, RecallAppendOutcome, TransactionMode}; - -/// Preserve the first copy of each key and record differing retries as Recall conflicts. -pub async fn publish_recall( - store: &ControlStore, - rows: &[RecallRow], -) -> Result, ControlError> { - let mut transaction = begin(store, TransactionMode::Write).await?; - let mut outcomes = Vec::with_capacity(rows.len()); - for row in rows { - let outcome; - (transaction, outcome) = append_recall(transaction, row).await?; - outcomes.push(outcome); - } - commit(transaction).await?; - Ok(outcomes) -} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs deleted file mode 100644 index ec9a4f533d..0000000000 --- a/crates/sqlbuild-orchestration/src/store/main/read_recall_events.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Stable cursor paging follows serialized committed appends, not producer timestamps. - -use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::store::types::Row; - -/// Page events in storage order; equal producer timestamps and late arrivals remain visible. -pub async fn read_recall_events( - store: &ControlStore, - after: i64, - limit: u32, -) -> Result, ControlError> { - let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query( - transaction, - "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", - vec![ - Parameter::Integer(after), - Parameter::Integer(i64::from(limit.min(1000))), - ], - ) - .await?; - commit(transaction).await?; - Ok(rows) -} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs index d92242c092..1228df7ae6 100644 --- a/crates/sqlbuild-orchestration/src/store/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -3,3 +3,7 @@ pub mod errors; pub mod main; pub mod models; pub(crate) mod types; + +#[cfg(test)] +#[path = "../../tests/store/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/tests/api.rs b/crates/sqlbuild-orchestration/tests/api.rs deleted file mode 100644 index 0b7016321c..0000000000 --- a/crates/sqlbuild-orchestration/tests/api.rs +++ /dev/null @@ -1,28 +0,0 @@ -#[path = "api/helpers.rs"] -mod helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "api/output/helpers.rs"] -mod output_helpers; -#[path = "api/recall/helpers.rs"] -mod recall_helpers; -#[path = "runs/helpers.rs"] -mod run_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "api/test_contract.rs"] -mod test_contract; -#[path = "api/test_location_health.rs"] -mod test_location_health; -#[path = "api/test_location_work.rs"] -mod test_location_work; -#[path = "api/test_reads.rs"] -mod test_reads; -#[path = "api/test_types.rs"] -mod test_types; -#[path = "api/test_worker.rs"] -mod test_worker; -#[path = "api/work/helpers.rs"] -mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index bb65bdcf07..9743c15603 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -1,9 +1,8 @@ use std::time::Duration; -use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::store::models::ControlStore; - -use crate::test_types::TestApi; +use crate::api::main::serve_api::serve_api; +use crate::api::tests::test_types::TestApi; +use crate::store::models::ControlStore; pub(crate) async fn start_api(store: &ControlStore) -> TestApi { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs index 230312a314..6d9f455567 100644 --- a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -1,11 +1,11 @@ //! Worker output contract helpers inspect durable bytes on both databases. +use crate::runs::models::OutputPublication; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; use serde_json::json; -use sqlbuild_orchestration::runs::models::OutputPublication; -use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; -use sqlbuild_orchestration::store::models::ControlStore; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::models::WorkerClient; use sqlbuild_recall::records::models::{OmittedOutput, OutputChunk, OutputStream}; use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs index edc555b357..d6bf078a67 100644 --- a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs @@ -2,12 +2,12 @@ use std::time::Duration; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; use serde_json::json; -use sqlbuild_orchestration::runs::models::{RecallBatch, RecallRecord}; -use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; -use sqlbuild_orchestration::store::models::ControlStore; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::models::WorkerClient; use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::event_object_row::event_object_row; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; @@ -20,7 +20,7 @@ use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; use sqlbuild_recall::writer::models::SessionConfig; use uuid::Uuid; -use crate::output_helpers::stored_bytes; +use crate::api::tests::output_helpers::stored_bytes; fn record(row: RecallRow) -> RecallRecord { RecallRecord { diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index be3b784298..4b311bbb1b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -1,34 +1,32 @@ use std::collections::BTreeMap; use std::time::Duration; +use crate::api::errors::ClientError; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::next_remote_generation::next_remote_generation; +use crate::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::main::migrate_store::migrate_store; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; -use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::main::verify_execution::verify_execution; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::run_helpers::{queue, request, snapshot}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::ApiContractTestCase; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::ApiContractTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() @@ -436,7 +434,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie ); } } - let mut forged = json!(crate::output_helpers::chunk( + let mut forged = json!(crate::api::tests::output_helpers::chunk( 0, sqlbuild_recall::records::models::OutputStream::Stdout, b"forged" @@ -502,7 +500,7 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie let terminal = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::Finished { - outcome: sqlbuild_orchestration::runs::models::RunOutcome::Cancelled, + outcome: crate::runs::models::RunOutcome::Cancelled, reason: "worker acknowledged cancellation".into(), exit_code: Some(143), }, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs new file mode 100644 index 0000000000..d141d28f37 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -0,0 +1,22 @@ +//! HTTP contracts compiled within the API owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "output/helpers.rs"] +mod output_helpers; +#[path = "recall/helpers.rs"] +mod recall_helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_location_health.rs"] +mod test_location_health; +#[path = "test_location_work.rs"] +mod test_location_work; +#[path = "test_reads.rs"] +mod test_reads; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_worker.rs"] +mod test_worker; +#[path = "work/helpers.rs"] +mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs index 38928875b7..002eed46e0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -1,23 +1,21 @@ +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::report_remote_location_health::report_remote_location_health; +use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; -use sqlbuild_orchestration::platform::models::{ - LocationHealth, LocationHealthObservation, LocationHealthReport, -}; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::run_helpers::snapshot; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::LocationHealthTestCase; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationHealthTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_scoped_location_observations_when_retried_or_read_then_history_and_authority_are_preserved() diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 52b883742f..03d21e27b5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -1,27 +1,27 @@ -use chrono::{Duration, Utc}; -use reqwest::StatusCode; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; -use sqlbuild_orchestration::platform::main::location_work::location_work; -use sqlbuild_orchestration::platform::main::report_location_health::report_location_health; -use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; -use sqlbuild_orchestration::platform::models::{ +use crate::api::errors::ClientError; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, LocationWorkRecord, LocationWorkRequest, LocationWorkResult, }; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use chrono::{Duration, Utc}; +use reqwest::StatusCode; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::run_helpers::snapshot; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::LocationWorkTestCase; -use crate::work_helpers::execute; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::api::tests::work_helpers::execute; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable_results_are_enforced() diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index 598dbd7496..1a3b7a06b9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -3,38 +3,37 @@ use std::collections::BTreeMap; use std::time::Duration; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::publish_recall; +use crate::worker::main::connect_worker::connect_worker; use chrono::Utc; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_orchestration::credentials::main::create_principal::create_principal; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; -use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::declarations::models::{TriggerChange, TriggerChangeRequest}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::publish_recall::publish_recall; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_recall::records::main::output_row::output_row; use sqlbuild_recall::records::models::{OutputStream, RowEnvelope}; use uuid::Uuid; -use crate::helpers::{read_json, start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::output_helpers::chunk; -use crate::recall_helpers::recall_contract; -use crate::run_helpers::{request, snapshot}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::ApiReadTestCase; +use crate::api::tests::helpers::{read_json, start_api, stop_api}; +use crate::api::tests::output_helpers::chunk; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::ApiReadTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_published_locations_and_scheduled_work_when_read_through_http_then_resources_and_histories_remain_scoped() diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index cf088cb040..133e475a53 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -1,7 +1,6 @@ -use sqlbuild_orchestration::runs::models::RunStatus; -use sqlbuild_orchestration::store::errors::ControlError; - -use crate::store_test_types::Backend; +use crate::runs::models::RunStatus; +use crate::store::errors::ControlError; +use crate::store::tests::store_test_types::Backend; pub(crate) struct ApiContractTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 60d5932881..e9d9e1aa80 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -1,36 +1,36 @@ use std::collections::BTreeMap; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::worker::errors::WorkerError; +use crate::worker::main::connect_worker::connect_worker; +use crate::worker::main::start_worker::start_worker; use axum::Router; use axum::response::Redirect; use axum::routing::get; use chrono::Utc; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{ - NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; -use sqlbuild_orchestration::worker::main::start_worker::start_worker; use uuid::Uuid; -use crate::helpers::{start_api, stop_api}; -use crate::location_helpers::create_location; -use crate::output_helpers::{chunk, output_contract}; -use crate::recall_helpers::recall_contract; -use crate::run_helpers::{queue, request, snapshot}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{ +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::output_helpers::{chunk, output_contract}; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::{ ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, }; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs index 32a7a3cbd3..2debfcd9b6 100644 --- a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -1,6 +1,6 @@ //! Fault injection changes only disposable test stores, never product time or lease semantics. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; pub(crate) async fn execute(store: &ControlStore, sql: &str) { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/credentials.rs b/crates/sqlbuild-orchestration/tests/credentials.rs deleted file mode 100644 index b31553bc0d..0000000000 --- a/crates/sqlbuild-orchestration/tests/credentials.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[path = "store/faults/helpers.rs"] -mod fault_helpers; -#[path = "store/faults/test_types.rs"] -mod fault_types; -#[path = "credentials/helpers.rs"] -mod helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "runs/helpers.rs"] -mod run_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "runs/termination/helpers.rs"] -mod terminal_helpers; -#[path = "credentials/test_contract.rs"] -mod test_contract; -#[path = "credentials/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index f86c76460c..4c28e90101 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -1,4 +1,4 @@ -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs index c6edcc2874..9a48e51758 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -1,8 +1,8 @@ //! Shared location fixtures use the same audited identity operation as operator bootstrap. -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; use uuid::Uuid; pub(crate) async fn create_location( @@ -10,7 +10,7 @@ pub(crate) async fn create_location( name: &str, actor: &str, ) -> Result { - sqlbuild_orchestration::credentials::main::create_location::create_location( + crate::credentials::main::create_location::create_location( store, name, &AuditContext { diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 67526d5c57..d26aa25639 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -1,26 +1,26 @@ use std::collections::BTreeMap; +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::main::create_location::create_location; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use chrono::{Duration, Utc}; -use sqlbuild_orchestration::credentials::main::authenticate_token::authenticate_token; -use sqlbuild_orchestration::credentials::main::create_location::create_location; -use sqlbuild_orchestration::credentials::main::create_principal::create_principal; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{location_state, stored_hashes}; -use crate::replica_helpers::second_store; -use crate::run_helpers::{queue, request, snapshot}; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::terminal_helpers::finish; -use crate::test_types::{CredentialTestCase, TransactionFault}; +use crate::credentials::tests::helpers::{location_state, stored_hashes}; +use crate::credentials::tests::test_types::{CredentialTestCase, TransactionFault}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs new file mode 100644 index 0000000000..9e6edc1c94 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -0,0 +1,10 @@ +//! Credential contracts compiled within the credential owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "locations/helpers.rs"] +pub(crate) mod location_helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index 077ee03ce9..c2efe4bf2a 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -1,5 +1,5 @@ -pub(crate) use crate::fault_types::TransactionFault; -use crate::store_test_types::Backend; +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; pub(crate) struct CredentialTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/platform.rs b/crates/sqlbuild-orchestration/tests/platform.rs deleted file mode 100644 index 3243ae357c..0000000000 --- a/crates/sqlbuild-orchestration/tests/platform.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "platform/helpers.rs"] -mod helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "platform/test_types.rs"] -mod test_types; -#[path = "platform/test_watchdog.rs"] -mod test_watchdog; diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index 310870ecfb..a00d9adf24 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -1,17 +1,17 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; +use crate::platform::models::PlatformEvent; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store::tests::helpers::read_recall_events; use chrono::{TimeDelta, Utc}; use serde_json::json; -use sqlbuild_orchestration::platform::models::PlatformEvent; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; -use sqlbuild_orchestration::store::models::ControlStore; use tokio::task::JoinHandle; use uuid::Uuid; -use crate::test_types::Observation; +use crate::platform::tests::test_types::Observation; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { let future = Utc::now() + TimeDelta::days(10); diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs new file mode 100644 index 0000000000..1a231c8e58 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -0,0 +1,8 @@ +//! Platform contracts compiled within the platform owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_watchdog.rs"] +mod test_watchdog; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 418d6a8cb3..01cf50f419 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -1,8 +1,8 @@ +use crate::platform::models::PlatformEvent; use chrono::{DateTime, Utc}; use serde::Deserialize; -use sqlbuild_orchestration::platform::models::PlatformEvent; -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; pub(crate) struct WatchdogTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index e119342ea7..0d03b0e495 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -1,19 +1,19 @@ use std::collections::BTreeSet; use std::time::Duration; -use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; -use sqlbuild_orchestration::platform::models::PlatformEvent; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::platform::main::run_watchdog::run_watchdog; +use crate::platform::models::PlatformEvent; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::{snapshot, wait_for_scans}; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::WatchdogTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{snapshot, wait_for_scans}; +use crate::platform::tests::test_types::WatchdogTestCase; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_durable_deduplicated_and_independent_of_leadership() diff --git a/crates/sqlbuild-orchestration/tests/runs.rs b/crates/sqlbuild-orchestration/tests/runs.rs deleted file mode 100644 index df23471f89..0000000000 --- a/crates/sqlbuild-orchestration/tests/runs.rs +++ /dev/null @@ -1,42 +0,0 @@ -#[path = "runs/claims/helpers.rs"] -mod claim_helpers; -#[path = "store/faults/helpers.rs"] -mod fault_helpers; -#[path = "store/faults/test_types.rs"] -mod fault_types; -#[path = "runs/helpers.rs"] -mod helpers; -#[path = "store/leases/helpers.rs"] -mod lease_helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "runs/monitor/helpers.rs"] -mod monitor_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "runs/termination/helpers.rs"] -mod terminal_helpers; -#[path = "runs/test_claims.rs"] -mod test_claims; -#[path = "runs/test_coalesce.rs"] -mod test_coalesce; -#[path = "runs/test_faults.rs"] -mod test_faults; -#[path = "runs/test_group_admission.rs"] -mod test_group_admission; -#[path = "runs/test_groups.rs"] -mod test_groups; -#[path = "runs/test_lifecycle.rs"] -mod test_lifecycle; -#[path = "runs/test_monitor.rs"] -mod test_monitor; -#[path = "runs/test_requests.rs"] -mod test_requests; -#[path = "runs/test_reruns.rs"] -mod test_reruns; -#[path = "runs/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs index 4fe22bb22f..6b2c0e2b1d 100644 --- a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -2,11 +2,11 @@ use std::collections::BTreeMap; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::models::{ +use crate::runs::main::change_run::change_run; +use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, }; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; use uuid::Uuid; pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 586e14c21e..c86ac7fe47 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -2,14 +2,27 @@ use std::collections::{BTreeMap, BTreeSet}; +use crate::runs::_helpers::requests::publication::publish_request; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::Overlap; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; use serde_json::json; -use sqlbuild_orchestration::runs::main::request_run::request_run; -use sqlbuild_orchestration::runs::models::{RunKind, RunOrigin, RunRequest}; -use sqlbuild_orchestration::scheduler::models::Overlap; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; +pub(crate) async fn request_run( + store: &ControlStore, + request: &RunRequest, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, applied) = publish_request(transaction, request).await?; + commit(transaction).await?; + Ok(applied) +} + pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { location_id, diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs index ec318744e7..c69347e480 100644 --- a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -2,17 +2,15 @@ use std::collections::BTreeMap; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; use chrono::{TimeDelta, Utc}; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest, -}; -use sqlbuild_orchestration::scheduler::models::SchedulerLease; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::helpers::queue; +use crate::runs::tests::helpers::queue; pub(crate) async fn start_run( store: &ControlStore, diff --git a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs index 9d6c347e0b..5b31b83c2f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -1,11 +1,9 @@ //! Terminal facts exercise claim release through the public run lifecycle. -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, -}; -use sqlbuild_orchestration::scheduler::models::SchedulerLease; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::runs::main::change_run::change_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; use uuid::Uuid; pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index ad747388df..d8c4151143 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -1,26 +1,26 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::RunStatus; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_run_claims::list_run_claims; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::RunStatus; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::claim_helpers::succeed; -use crate::helpers::{queue, request, snapshot}; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::terminal_helpers::finish; -use crate::test_types::{ +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::succeed; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::runs::tests::test_types::{ BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, }; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then_only_its_location_and_target_are_reserved() @@ -413,11 +413,9 @@ fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_in for id in batch { queue(store, &request(scope, *id, 0, &["orders"])).await; } - lease = sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease( - store, &lease, - ) - .await - .expect("keep leader live while preparing backlog"); + lease = crate::scheduler::main::renew_lease::renew_lease(store, &lease) + .await + .expect("keep leader live while preparing backlog"); } queue(store, &request(scope, 105, -1, &["customers"])).await; let started = std::time::Instant::now(); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index 3149b775d9..b9eb5090c4 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -1,21 +1,20 @@ use std::collections::BTreeMap; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::models::TickDecision; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use serde_json::json; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::scheduler::models::TickDecision; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::CoalescedRunTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::CoalescedRunTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs new file mode 100644 index 0000000000..530eb54c35 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -0,0 +1,28 @@ +#[path = "claims/helpers.rs"] +mod claim_helpers; +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "monitor/helpers.rs"] +mod monitor_helpers; +#[path = "termination/helpers.rs"] +pub(crate) mod terminal_helpers; +#[path = "test_claims.rs"] +mod test_claims; +#[path = "test_coalesce.rs"] +mod test_coalesce; +#[path = "test_faults.rs"] +mod test_faults; +#[path = "test_group_admission.rs"] +mod test_group_admission; +#[path = "test_groups.rs"] +mod test_groups; +#[path = "test_lifecycle.rs"] +mod test_lifecycle; +#[path = "test_monitor.rs"] +mod test_monitor; +#[path = "test_requests.rs"] +mod test_requests; +#[path = "test_reruns.rs"] +mod test_reruns; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 00d20c38c3..f784d6485b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -1,25 +1,25 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{queue, request, snapshot}; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{ClaimFailureTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ClaimFailureTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 8dc2a9e113..a57ce0a896 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -3,28 +3,28 @@ use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroU32; -use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::{ advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, request_job_run::request_job_run, }; -use sqlbuild_orchestration::runs::models::{JobRunRequest, RunDispatch, RunStatus}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::runs::models::{JobRunRequest, RunDispatch, RunStatus}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; use uuid::Uuid; -use crate::claim_helpers::{slot_counts, succeed}; -use crate::helpers::{queue, request, snapshot}; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{ +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ AdmissionParent, GroupAdmissionTestCase, GroupQueueTestCase, MixedAdmissionTestCase, RunLifecycleTestCase, }; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queue_order_win() { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index 6a502d207d..b2fe494aef 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -3,29 +3,28 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::{ advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, }; -use sqlbuild_orchestration::runs::models::{ +use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::{ - migrate_store::migrate_store, read_recall_events::read_recall_events, -}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use serde_json::json; use uuid::Uuid; -use crate::claim_helpers::{slot_counts, succeed}; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_and_parent_waits_for_terminal_evidence() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 5c23383e12..2f98d185eb 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -1,26 +1,26 @@ use std::collections::BTreeMap; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; -use sqlbuild_orchestration::runs::main::list_run_nodes::list_run_nodes; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::models::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{queue, request, snapshot}; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index 228dd090a2..62fe549bc8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -3,32 +3,31 @@ use std::collections::BTreeMap; use std::time::Duration; -use sqlbuild_orchestration::runs::main::change_run::change_run; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::heartbeat_run::heartbeat_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::main::monitor_runs::monitor_runs; -use sqlbuild_orchestration::runs::models::{ - RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; -use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; -use sqlbuild_orchestration::scheduler::models::SchedulerState; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus}; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, scheduler_cycle}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::{queue, request, snapshot}; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::monitor_helpers::{age_heartbeat, start_run}; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::monitor_helpers::{age_heartbeat, start_run}; +use crate::runs::tests::test_types::{ + MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_facts_retain_claims_and_allow_recovery() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 226cb57a5b..5ad011d62f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -2,24 +2,24 @@ use std::collections::BTreeMap; +use crate::credentials::models::AuditContext; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::{JobRunRequest, RunStatus}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; -use sqlbuild_orchestration::runs::models::{JobRunRequest, RunStatus}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_and_frozen_execution_commit_once() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 0729862d1f..4c50700634 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -2,28 +2,30 @@ use std::collections::{BTreeMap, BTreeSet}; -use serde_json::json; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::runs::main::{ +use crate::credentials::models::AuditContext; +use crate::runs::main::{ advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, }; -use sqlbuild_orchestration::runs::models::{ +use crate::runs::models::{ JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, RunOutcome, RunStatus, }; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; use uuid::Uuid; -use crate::claim_helpers::{slot_counts, succeed}; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::helpers::snapshot; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::{RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{ + RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 0646241008..fc108dd8c5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -2,10 +2,10 @@ use std::collections::BTreeSet; -use sqlbuild_orchestration::runs::models::{RerunRequest, RunStatus}; +use crate::runs::models::{RerunRequest, RunStatus}; -pub(crate) use crate::fault_types::TransactionFault; -use crate::store_test_types::Backend; +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; pub(crate) struct ScopedClaimTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/scheduler.rs b/crates/sqlbuild-orchestration/tests/scheduler.rs deleted file mode 100644 index 774474ceef..0000000000 --- a/crates/sqlbuild-orchestration/tests/scheduler.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[path = "store/faults/helpers.rs"] -mod fault_helpers; -#[path = "store/faults/test_types.rs"] -mod fault_types; -#[path = "scheduler/helpers.rs"] -mod helpers; -#[path = "store/leases/helpers.rs"] -mod lease_helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "scheduler/test_contract.rs"] -mod test_contract; -#[path = "scheduler/test_types.rs"] -mod test_types; -#[path = "scheduler/ticks/test_ticks.rs"] -mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index 8cac2b9924..a1e8ee33dc 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -1,15 +1,17 @@ -//! Contention and expiry exercise public APIs against the real databases, not fake clocks. +//! Contention and expiry exercise scheduler ownership against real databases, not fake clocks. -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +pub(crate) use crate::scheduler::main::_acquire_lease::acquire_lease; +pub(crate) use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +pub(crate) use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::lease_helpers::expire_lease; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; pub(crate) async fn leadership_contract(backend: Backend) -> Option { let test_store = open_test_store(backend).await?; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs index 084f5fbb10..13bdfd4250 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs @@ -1,7 +1,7 @@ -use crate::helpers::leadership_contract; -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::test_types::LeadershipTestCase; +use crate::scheduler::tests::contracts::helpers::leadership_contract; +use crate::scheduler::tests::contracts::test_types::LeadershipTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; #[test] fn given_two_schedulers_when_leadership_expires_then_only_the_new_epoch_can_renew() { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs new file mode 100644 index 0000000000..57d98cb7cf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -0,0 +1,10 @@ +//! Real-store scheduler contracts compiled within the scheduler owner. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; +#[path = "ticks/test_ticks.rs"] +mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index c7db9ff190..fd5543086b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -1,6 +1,6 @@ //! Leadership scenarios run independently on both supported engines. -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; #[derive(Debug)] pub(crate) struct LeadershipTestCase { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 7d4bac4e82..223bc36025 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -2,28 +2,28 @@ use std::collections::BTreeMap; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickOutcome; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; use serde_json::{Value, json}; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::runs::main::list_runs::list_runs; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::scheduler::main::list_ticks::list_ticks; -use sqlbuild_orchestration::scheduler::models::TickOutcome; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; -use crate::ticks::test_types::TransactionFault; +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::test_types::TransactionFault; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs index af4492a29d..91d523b69f 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs @@ -1,9 +1,9 @@ //! Crash points must not separate a tick from its queued work and durable cursor. -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::ticks::helpers::transaction_contract; -use crate::ticks::test_types::TickTransactionTestCase; +use crate::scheduler::tests::contracts::ticks::helpers::transaction_contract; +use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; #[test] fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index e62bf51595..382d0f17bf 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -2,20 +2,22 @@ use std::time::Duration; -use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; -use sqlbuild_orchestration::scheduler::main::scheduler_cycle::scheduler_cycle; -use sqlbuild_orchestration::scheduler::models::SchedulerState; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::run_scheduler::run_scheduler; +use crate::scheduler::models::SchedulerState; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::lease_helpers::expire_lease; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::ticks::helpers::snapshot; -use crate::ticks::test_types::{IdleLauncher, TickTransactionTestCase}; +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::scheduler::tests::contracts::ticks::test_types::{ + IdleLauncher, TickTransactionTestCase, +}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() @@ -75,13 +77,10 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain .1 .leader ); - let runs = sqlbuild_orchestration::runs::main::list_runs::list_runs( - store, - (location, "development"), - 100, - ) - .await - .expect("queued run"); + let runs = + crate::runs::main::list_runs::list_runs(store, (location, "development"), 100) + .await + .expect("queued run"); assert_eq!(runs.len(), test_case.expected_runs); let (stop, shutdown) = tokio::sync::watch::channel(false); let (_changed, changes) = tokio::sync::watch::channel(()); diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 34e9c4b198..1dd53ec524 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -1,24 +1,24 @@ //! Operator trigger facts remain consistent across publication and retry boundaries. -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; -use sqlbuild_orchestration::declarations::main::list_trigger_events::list_trigger_events; -use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; -use sqlbuild_orchestration::declarations::models::{ - TriggerChange, TriggerChangeRequest, TriggerStatus, -}; -use sqlbuild_orchestration::scheduler::main::acquire_lease::acquire_lease; -use sqlbuild_orchestration::scheduler::main::evaluate_trigger::evaluate_trigger; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; -use crate::fault_helpers::{install_fault, remove_fault}; -use crate::location_helpers::create_location; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::ticks::helpers::{snapshot, trigger_audits}; -use crate::ticks::test_types::{TransactionFault, TriggerControlsTestCase}; +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::{snapshot, trigger_audits}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + TransactionFault, TriggerControlsTestCase, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; #[test] fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 5f10deea0d..d523cedb8e 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -1,7 +1,7 @@ //! Database cases and expected scheduler tick outcomes. -pub(crate) use crate::fault_types::TransactionFault; -use crate::store_test_types::Backend; +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; pub(crate) struct TickTransactionTestCase { pub(crate) description: &'static str, @@ -18,20 +18,20 @@ pub(crate) struct TriggerControlsTestCase { pub(crate) struct IdleLauncher; -impl sqlbuild_orchestration::scheduler::types::RunLauncher for IdleLauncher { +impl crate::scheduler::types::RunLauncher for IdleLauncher { async fn poll( &mut self, - _store: &sqlbuild_orchestration::store::models::ControlStore, - _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, - ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { Ok(()) } async fn shutdown( &mut self, - _store: &sqlbuild_orchestration::store::models::ControlStore, - _lease: Option<&sqlbuild_orchestration::scheduler::models::SchedulerLease>, - ) -> Result<(), sqlbuild_orchestration::store::errors::ControlError> { + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { Ok(()) } } diff --git a/crates/sqlbuild-orchestration/tests/snapshots.rs b/crates/sqlbuild-orchestration/tests/snapshots.rs deleted file mode 100644 index 13be1a8abc..0000000000 --- a/crates/sqlbuild-orchestration/tests/snapshots.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "snapshots/test_contract.rs"] -mod contract; -#[path = "snapshots/helpers.rs"] -mod helpers; -#[path = "credentials/locations/helpers.rs"] -mod location_helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "snapshots/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index e486bb4182..7c093b292f 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -2,21 +2,21 @@ use std::collections::BTreeMap; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; use serde_json::json; -use sqlbuild_orchestration::snapshots::main::current_snapshot::current_snapshot; -use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; -use crate::location_helpers::create_location; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; -use crate::test_types::SchemaRejectionCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::snapshots::tests::test_types::SchemaRejectionCase; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs index 8096938089..708a6aff32 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -1,7 +1,7 @@ -use crate::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::test_types::{RecoveryTestCase, SnapshotTestCase}; +use crate::snapshots::tests::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; +use crate::snapshots::tests::test_types::{RecoveryTestCase, SnapshotTestCase}; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; #[test] fn given_control_store_when_snapshots_publish_then_generations_sources_and_changes_are_consistent() diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs new file mode 100644 index 0000000000..6963cfddf6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -0,0 +1,8 @@ +//! Snapshot contracts compiled within the snapshot owner. + +#[path = "test_contract.rs"] +mod contract; +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs index cd4379f770..61a2c6832b 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -1,6 +1,6 @@ //! Cases that alter a disposable schema to verify startup rejection. -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; #[derive(Debug)] pub(super) struct SchemaRejectionCase { diff --git a/crates/sqlbuild-orchestration/tests/store.rs b/crates/sqlbuild-orchestration/tests/store.rs deleted file mode 100644 index 5a79851f21..0000000000 --- a/crates/sqlbuild-orchestration/tests/store.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Recall storage integration contracts for SQLite and PostgreSQL. - -#[path = "store/recall/helpers.rs"] -mod helpers; -#[path = "store/replicas/helpers.rs"] -mod replica_helpers; -#[path = "store/helpers.rs"] -mod store_helpers; -#[path = "store/test_types.rs"] -mod store_test_types; -#[path = "store/recall/test_contract.rs"] -mod test_contract; -#[path = "store/recall/test_types.rs"] -mod test_types; -#[path = "store/recall/test_views.rs"] -mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs index c7d5c81dd5..8c221edcc0 100644 --- a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -1,8 +1,8 @@ //! Database-trigger fault injection at durable publication boundaries. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; -use crate::fault_types::TransactionFault; +use crate::store::tests::fault_types::TransactionFault; pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index f93f44f0f5..db2525d17d 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -1,10 +1,10 @@ //! Real-database setup shared by storage, scheduling, and queue contracts. -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; use uuid::Uuid; -use crate::store_test_types::{Backend, TestStore}; +use crate::store::tests::store_test_types::{Backend, TestStore}; pub(crate) fn run(future: impl std::future::Future) -> T { tokio::runtime::Builder::new_current_thread() diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs index b813d957d3..59f067dc23 100644 --- a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -1,6 +1,6 @@ //! Expire the persisted lease without introducing a fake control-store clock. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; pub(crate) async fn expire_lease(store: &ControlStore) { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs index 2fb0605a1d..a530b60e5c 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs @@ -1,16 +1,55 @@ //! Recall publication fixtures exercise the control store's durable append boundary. +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, RecallAppendOutcome, TransactionMode}; +use crate::store::types::Row; use serde_json::json; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::publish_recall::publish_recall; -use sqlbuild_orchestration::store::main::read_recall_events::read_recall_events; use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::event_object_row::event_object_row; use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; -use crate::replica_helpers::second_store; -use crate::store_helpers::{close_store, close_test_store, open_test_store}; -use crate::store_test_types::Backend; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) async fn publish_recall( + store: &ControlStore, + rows: &[RecallRow], +) -> Result, ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut outcomes = Vec::with_capacity(rows.len()); + for row in rows { + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + outcomes.push(outcome); + } + commit(transaction).await?; + Ok(outcomes) +} + +pub(crate) async fn read_recall_events( + store: &ControlStore, + after: i64, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows) +} pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { event_object_row(&RowEnvelope { invocation_id: "orders-build".into(), diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs index d0ccf95c46..988e9542be 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -1,9 +1,9 @@ -//! Idempotency and commit-order paging through the public store API. +//! Idempotency and commit-order paging through real store transactions. -use crate::helpers::recall_contract; -use crate::store_helpers::run; -use crate::store_test_types::Backend; -use crate::test_types::RecallTestCase; +use crate::store::tests::helpers::recall_contract; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallTestCase; #[test] fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs index 481cac0601..53cbf105a6 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -1,6 +1,6 @@ //! Expected Recall storage outcomes for both supported databases. -use crate::store_test_types::Backend; +use crate::store::tests::store_test_types::Backend; pub(crate) struct RecallTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index 75a8b9971d..a157ecf787 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -1,9 +1,8 @@ //! Durable Recall views preserve the published execution records. +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::read_recall_view::read_recall_view; use serde_json::json; -use sqlbuild_orchestration::store::main::migrate_store::migrate_store; -use sqlbuild_orchestration::store::main::publish_recall::publish_recall; -use sqlbuild_orchestration::store::main::read_recall_view::read_recall_view; use sqlbuild_recall::records::main::artifact_rows::artifact_rows; use sqlbuild_recall::records::main::delivery_row::delivery_row; use sqlbuild_recall::records::main::event_object_row::event_object_row; @@ -14,9 +13,10 @@ use sqlbuild_recall::records::models::{ ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, }; -use crate::store_helpers::{close_test_store, open_test_store, run}; -use crate::store_test_types::Backend; -use crate::test_types::RecallViewTestCase; +use crate::store::tests::helpers::publish_recall; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallViewTestCase; #[test] fn given_native_context_output_and_audits_when_reading_views_then_both_databases_preserve_their_meaning() diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs index 22fe1b374b..443b0e5e26 100644 --- a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -1,7 +1,7 @@ //! Independent database handles for multi-replica integration contracts. -use sqlbuild_orchestration::store::main::open_store::open_store; -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { let ControlStore::Postgres(pool) = store else { diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs new file mode 100644 index 0000000000..a257c9aeb9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -0,0 +1,22 @@ +//! Recall storage integration contracts for SQLite and PostgreSQL. + +#[path = "faults/helpers.rs"] +pub(crate) mod fault_helpers; +#[path = "faults/test_types.rs"] +pub(crate) mod fault_types; +#[path = "recall/helpers.rs"] +pub(crate) mod helpers; +#[path = "leases/helpers.rs"] +pub(crate) mod lease_helpers; +#[path = "replicas/helpers.rs"] +pub(crate) mod replica_helpers; +#[path = "helpers.rs"] +pub(crate) mod store_helpers; +#[path = "test_types.rs"] +pub(crate) mod store_test_types; +#[path = "recall/test_contract.rs"] +mod test_contract; +#[path = "recall/test_types.rs"] +mod test_types; +#[path = "recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs index 55a1a65092..1ffddfd679 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -1,6 +1,6 @@ //! Store fixtures retain their isolated SQLite directory or PostgreSQL schema until cleanup. -use sqlbuild_orchestration::store::models::ControlStore; +use crate::store::models::ControlStore; #[derive(Clone, Copy, Debug)] pub(crate) enum Backend { From cd7a0def2d1fd7517ea4a1007fd774244de90cf4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:46:35 +0000 Subject: [PATCH 041/138] feat: execute explicit backfill windows Freeze inclusive typed cursor windows with run requests, derive execution overrides without mutating pinned job settings, and preserve the window through group children and reruns. Verify real local and Docker warehouse writes and the SQLite/PostgreSQL publication contracts. --- .../orchestration/backfill_integer.sql | 16 ++ .../orchestration/backfill_timestamp.sql | 17 ++ .../sqlbuild-cli/src/argv/command_table.json | 28 +++ .../src/native_commands/_helpers/decision.rs | 47 ++++- .../src/runs/_helpers/requests/job.rs | 7 +- .../src/runs/main/request_job_run.rs | 10 +- .../sqlbuild-orchestration/src/runs/models.rs | 18 ++ .../src/scheduler/_helpers/persistence.rs | 1 + .../tests/runs/helpers.rs | 1 + .../tests/runs/test_group_admission.rs | 5 + .../tests/runs/test_groups.rs | 2 + .../tests/runs/test_requests.rs | 29 +++- .../tests/runs/test_reruns.rs | 7 +- .../_helpers/location_command/worker.rs | 23 ++- .../publication/_test_types.py | 7 + .../publication/test_backfill.py | 164 ++++++++++++++++++ 16 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql create mode 100644 benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql new file mode 100644 index 0000000000..2d882a92ae --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql @@ -0,0 +1,16 @@ +MODEL ( + description 'Backfill orders over an explicit integer window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor batch_id, + cursor_type integer, + cursor_start 0, + cursor_watermark_mode all, + cursor_inputs (raw_orders (column batch_id, roles [filter, watermark])), + batch_size '2', + batch_concurrency 1, +); +SELECT amount, batch_id FROM __source("raw_orders") +WHERE batch_id >= __cursor_start() AND batch_id < __cursor_end() diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql new file mode 100644 index 0000000000..fa8b2bcd25 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql @@ -0,0 +1,17 @@ +MODEL ( + description 'Backfill orders over an explicit timestamp window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor ordered_at, + cursor_type timestamp, + cursor_grain day, + cursor_start '2026-07-01', + cursor_watermark_mode all, + cursor_inputs (raw_orders (column ordered_at, roles [filter, watermark])), + batch_size 1d, + batch_concurrency 1, +); +SELECT amount, ordered_at FROM __source("raw_orders") +WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index fea155edee..038ceefb5b 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -583,6 +583,34 @@ "help": "run UUID; reuse it with identical arguments after an uncertain result", "group": 1 }, + { + "kind": "store", + "options": ["--start-cursor-ts"], + "dest": "admin_start_cursor_ts", + "help": "backfill timestamp start (RFC3339); requires --end-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-ts"], + "dest": "admin_end_cursor_ts", + "help": "inclusive backfill timestamp end (RFC3339); requires --start-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--start-cursor-int"], + "dest": "admin_start_cursor_int", + "help": "backfill integer start; requires --end-cursor-int", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-int"], + "dest": "admin_end_cursor_int", + "help": "inclusive backfill integer end; requires --start-cursor-int", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index c0e69b8143..f78d6f1874 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -8,7 +8,7 @@ use serde_json::json; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; -use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; use sqlbuild_orchestration::store::errors::ControlError; use uuid::Uuid; @@ -125,6 +125,50 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result None, + (Some(start), Some(end), None, None) => Some(RunWindow::Timestamp { + start: DateTime::parse_from_rfc3339(&start) + .map_err(|_| { + ControlError::Invalid( + "backfill start must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + end: DateTime::parse_from_rfc3339(&end) + .map_err(|_| { + ControlError::Invalid( + "backfill end must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + }), + (None, None, Some(start), Some(end)) => Some(RunWindow::Integer { + start: start.parse().map_err(|_| { + ControlError::Invalid( + "backfill start must be a signed 64-bit integer".into(), + ) + })?, + end: end.parse().map_err(|_| { + ControlError::Invalid( + "backfill end must be a signed 64-bit integer".into(), + ) + })?, + }), + _ => { + return Err(ControlError::Invalid( + "backfill requires both start and end of exactly one cursor type" + .into(), + )); + } + }; return Ok(ControlAdmin::RequestJobRun(JobRunRequest { run_id: event_id, location_id: id, @@ -132,6 +176,7 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result start > end, + RunWindow::Integer { start, end } => start > end, + }) { + return Err(ControlError::Invalid( + "backfill window start must not follow end".into(), + )); + } let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); let transaction = begin(store, TransactionMode::Write).await?; let (transaction, retry) = request_retry( diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index c049379a61..64ab1a370c 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -73,6 +73,20 @@ impl RunStatus { } } +/// Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunWindow { + Timestamp { + start: DateTime, + end: DateTime, + }, + Integer { + start: i64, + end: i64, + }, +} + /// The authorized selection and pinned execution inputs are immutable once requested. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRequest { @@ -94,6 +108,8 @@ pub struct RunRequest { pub selection: BTreeSet, pub write_set: BTreeSet, pub settings: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, pub concurrency_groups: BTreeSet, pub overlap: crate::scheduler::models::Overlap, } @@ -108,6 +124,8 @@ pub struct JobRunRequest { pub job_key: String, pub reason: String, pub priority: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, } /// A retry derives its execution scope from a terminal run, never from caller selectors. diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 3a69b46ea5..0f659487c0 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -61,6 +61,7 @@ pub(crate) async fn persist_ticks( write_set.clone() }, settings: trigger.job_json.settings.clone(), + window: None, concurrency_groups: trigger.job_json.concurrency_groups.clone(), overlap: trigger.job_json.overlap, }; diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index c86ac7fe47..12663d3921 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -80,6 +80,7 @@ pub(crate) fn request( write_set: selection.clone(), selection, settings: BTreeMap::new(), + window: None, concurrency_groups: BTreeSet::new(), overlap: Overlap::Allow, } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index a57ce0a896..015e0ca74c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -90,6 +90,7 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu job_key: "daily".into(), reason: "Exercise mixed queue ordering".into(), priority: test_case.parent_priority, + window: None, }, ) .await @@ -184,6 +185,7 @@ fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknow job_key: "daily".into(), reason: "Exercise launcher eligibility".into(), priority: 1, + window: None, }, ) .await @@ -426,6 +428,7 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th job_key: format!("orders_group_{index}"), reason: "Exercise capacity-aware admission".into(), priority: index as i64 + 1, + window: None, }, ) .await @@ -585,6 +588,7 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t job_key: "daily".into(), reason: "Older group".into(), priority: 1, + window: None, }; request_job_run(store, &actor, &older) .await @@ -758,6 +762,7 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate job_key: format!("group_{index}"), reason: "Exercise blocked group queue".into(), priority: 1, + window: None, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index b2fe494aef..10f1b53bf8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -78,6 +78,7 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a job_key: "daily_orders".into(), reason: "Build the group".into(), priority: 1, + window: None, }; assert!( request_job_run(store, &context, &request) @@ -351,6 +352,7 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups job_key: job_key.into(), reason: "Build independent groups".into(), priority: 1, + window: None, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index 5ad011d62f..df6994d8cf 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -6,7 +6,7 @@ use crate::credentials::models::AuditContext; use crate::runs::main::claim_next_run::claim_next_run; use crate::runs::main::list_runs::list_runs; use crate::runs::main::request_job_run::request_job_run; -use crate::runs::models::{JobRunRequest, RunStatus}; +use crate::runs::models::{JobRunRequest, RunOrigin, RunStatus, RunWindow}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; @@ -68,6 +68,10 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an job_key: "daily_orders".into(), reason: "Refresh orders".into(), priority: 7, + window: Some(RunWindow::Timestamp { + start: "2026-07-03T01:17:23+05:30".parse().expect("start"), + end: "2026-07-04T01:17:23+05:30".parse().expect("end"), + }), }; for invalid in [ JobRunRequest { @@ -86,6 +90,17 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an reason: " ".into(), ..request.clone() }, + JobRunRequest { + window: Some(RunWindow::Integer { start: 11, end: 3 }), + ..request.clone() + }, + JobRunRequest { + window: Some(RunWindow::Timestamp { + start: "2026-07-04T01:17:23Z".parse().expect("start"), + end: "2026-07-03T01:17:23Z".parse().expect("end"), + }), + ..request.clone() + }, ] { assert!(request_job_run(store, &context, &invalid).await.is_err()); } @@ -139,6 +154,9 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an let mut changed = request.clone(); changed.reason = "Different request".into(); assert!(request_job_run(store, &context, &changed).await.is_err()); + changed = request.clone(); + changed.window = Some(RunWindow::Integer { start: 3, end: 11 }); + assert!(request_job_run(store, &context, &changed).await.is_err()); let other_actor = AuditContext { actor: "other-operator".into(), ..context.clone() @@ -168,6 +186,14 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an .expect("queued request"); assert_eq!(claim.request.run_id, request.run_id); assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!(claim.request.origin, RunOrigin::Backfill); + assert_eq!( + claim.request.window, + Some(RunWindow::Timestamp { + start: "2026-07-02T19:47:23Z".parse().expect("normalized start"), + end: "2026-07-03T19:47:23Z".parse().expect("normalized end"), + }) + ); assert_eq!( claim.request.image_id.as_deref(), Some("sha256:synthetic-orders-image") @@ -189,6 +215,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an assert_eq!(claim.request.requested_by, "test-operator"); let fresh = JobRunRequest { run_id: Uuid::now_v7(), + window: Some(RunWindow::Integer { start: 7, end: 7 }), ..request.clone() }; assert!( diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 4c50700634..75339ea1fe 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -9,7 +9,7 @@ use crate::runs::main::{ }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, - RunOutcome, RunStatus, + RunOutcome, RunStatus, RunWindow, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -80,6 +80,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ job_key: "daily_orders".into(), reason: "Build orders".into(), priority: 1, + window: Some(RunWindow::Integer { start: -3, end: 11 }), }, ) .await @@ -285,6 +286,10 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ assert_eq!(claim.request.origin, RunOrigin::Rerun); assert_eq!(claim.request.priority, 7); assert_eq!(claim.request.settings, settings[expected_job]); + assert_eq!( + claim.request.window, + Some(RunWindow::Integer { start: -3, end: 11 }) + ); selected.extend(claim.request.selection.iter().cloned()); succeed(store, &claim).await; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 52da11300d..b7876f07f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -10,7 +10,7 @@ use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_orchestration::runs::models::RunOutcome; +use sqlbuild_orchestration::runs::models::{RunOutcome, RunWindow}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; use crate::bindings::_helpers::build_command::command::run_build; @@ -31,8 +31,27 @@ fn build_request<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; + let mut settings = frozen.settings; + if let Some(window) = frozen.window { + for key in [ + "start_cursor", + "end_cursor", + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] { + settings.remove(key); + } + let (kind, start, end) = match window { + RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), + RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), + }; + settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); + settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); + } let mut argv: Vec> = std::iter::once("build".to_owned()) - .chain(job_settings_arguments(&frozen.settings).map_err(PyValueError::new_err)?) + .chain(job_settings_arguments(&settings).map_err(PyValueError::new_err)?) .map(|argument| PyString::new(py, &argument)) .collect(); argv.extend([ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 135c64aaf6..e6b239fffe 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -120,3 +120,10 @@ class WorkerRestartTestCase(WorkerExecutionTestCase): description: str shutdown_signal: int = 15 expected_scheduler_exit: int = 0 + + +@dataclass(frozen=True) +class BackfillExecutionTestCase(WorkerExecutionTestCase): + description: str + window_arguments: tuple[str, ...] = () + expected_window: dict[str, Any] = field(default_factory=dict) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py new file mode 100644 index 0000000000..9d54256fc6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -0,0 +1,164 @@ +"""Explicit backfills and their reruns execute the same pinned window through real workers.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + BackfillExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + BackfillExecutionTestCase( + description="local integer window overrides authored bounds and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql" + ).read_text() + }, + window_arguments=("--start-cursor-int", "3", "--end-cursor-int", "6"), + expected_window={"kind": "integer", "start": 3, "end": 6}, + ), + BackfillExecutionTestCase( + description="Docker timestamp window normalizes offsets and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + launcher="docker", + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql" + ).read_text() + }, + window_arguments=( + "--start-cursor-ts", + "2026-07-03T05:30:00+05:30", + "--end-cursor-ts", + "2026-07-04T05:30:00+05:30", + ), + expected_window={ + "kind": "timestamp", + "start": "2026-07-03T00:00:00Z", + "end": "2026-07-04T00:00:00Z", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_only_its_frozen_window_is_written( + test_case: BackfillExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files=test_case.files + | { + "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "orders = job('orders', description='Windowed orders.', select='orders', " + "settings={'start_cursor': '0', 'end_cursor': '1'})\n" + "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute( + "CREATE TABLE raw_orders AS SELECT * FROM (VALUES (11, 2, TIMESTAMP '2026-07-02'), (23, 3, TIMESTAMP '2026-07-03'), (37, 6, TIMESTAMP '2026-07-04'), (41, 7, TIMESTAMP '2026-07-05')) t(amount, batch_id, ordered_at)" + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + run_id: str = str(uuid4()) + arguments: list[str] = [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + "annual_orders", + "--target", + "development", + "--run-id", + run_id, + "--reason", + "Backfill an explicit order window", + *test_case.window_arguments, + ] + rejected: subprocess.CompletedProcess[str] = subprocess.run( + arguments[:-2], capture_output=True, text=True, timeout=10, check=False + ) + assert rejected.returncode != 0 + assert "both start and end of exactly one cursor type" in rejected.stderr + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + for expected in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + arguments, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + warehouse.execute("UPDATE raw_orders SET amount=amount+100") + retry_id: str = str(uuid4()) + retried: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + run_id, + "--run-id", + retry_id, + "--reason", + "Repeat the same explicit order window", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert retried.returncode == 0, retried.stderr + retry: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=retry_id + ) + assert retry["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(123,), (137,)] + with sqlite3.connect(worker.control_store) as control: + requests: list[dict[str, Any]] = [ + json.loads(row[0]) + for row in control.execute("SELECT request_json FROM sqb_runs").fetchall() + ] + assert len(requests) == 4 + assert all(request["window"] == test_case.expected_window for request in requests) + assert {request["origin"] for request in requests} == {"backfill", "rerun"} + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='job.run_requested'" + ).fetchone() == (1,) From caefe0bf3b8b015043003f385b19be5227e80fa1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:34:20 +0000 Subject: [PATCH 042/138] feat: publish and refresh authored calendars --- Cargo.lock | 2 + .../orchestration/calendar_orders.py | 17 ++ .../src/native_commands/_helpers/execution.rs | 15 +- .../src/native_commands/main/_run_location.rs | 7 +- .../src/native_commands/models.rs | 10 +- .../declarations/main/publish_definitions.rs | 44 ++- .../src/declarations/models.rs | 1 + .../tests/scheduler/ticks/test_triggers.rs | 123 ++++++++ crates/sqlbuild-python/Cargo.toml | 2 + .../_helpers/compiled_project/snapshot.rs | 16 +- .../compiled_project/snapshot_declarations.rs | 104 ++----- .../_helpers/location_command/command.rs | 32 +- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_declarations.rs | 84 +++++- .../native_host/orchestration_triggers.rs | 243 +++++++++++++++ fensu.toml | 1 + src/sqlbuild/orchestration.py | 22 +- .../main/compile_snapshot.py | 4 +- .../orchestration_declarations/models.py | 32 +- .../publication/_test_types.py | 15 + .../publication/test_calendar.py | 285 ++++++++++++++++++ 21 files changed, 934 insertions(+), 126 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py diff --git a/Cargo.lock b/Cargo.lock index d3c2ae6c05..8a5982d252 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2631,6 +2631,8 @@ dependencies = [ name = "sqlbuild-python" version = "0.186.2" dependencies = [ + "chrono", + "chrono-tz", "fensu-policy", "mimalloc", "pyo3", diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py new file mode 100644 index 0000000000..7beff48f02 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -0,0 +1,17 @@ +"""Provider-backed fulfillment times evaluated in the code location, never by the scheduler.""" + +from sqlbuild.orchestration import calendar_schedule, job + +orders = job("orders", description="Build order reporting.", select="orders") + + +@calendar_schedule( + job=orders, + description="Refresh orders at upcoming fulfillment times.", + horizon="8s", + timezone="America/New_York", + catch_up="all", + max_catch_up=10, +) +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 7de7f07935..ee24726afd 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -6,11 +6,11 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::{Duration, SystemTime}; +use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; -use sqlbuild_orchestration::snapshots::models::SnapshotRecord; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; @@ -45,8 +45,8 @@ use crate::native_commands::constants::{ use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, - LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationPublication, + LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; use crate::workers::main::open_workers::open_workers; use crate::workers::models::Workers; @@ -57,7 +57,7 @@ pub(crate) async fn compile_generation( generation: i64, sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], stopped: impl Future>, -) -> Result, ControlError> { +) -> Result>)>, ControlError> { let (reply, publication) = oneshot::channel(); requests .send(LocationCompileRequest { @@ -89,7 +89,8 @@ pub(crate) async fn compile_generation( } }; let published = record.is_some(); - if let Some(record) = record { + let refresh_at = record.as_ref().and_then(|(_, refresh_at)| *refresh_at); + if let Some((record, _)) = record { let output = serde_json::to_string(&record) .map_err(|error| ControlError::Invalid(error.to_string()))?; if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { @@ -99,14 +100,14 @@ pub(crate) async fn compile_generation( ))); } } - Ok(Some(published)) + Ok(Some((published, refresh_at))) } /// The child is retained by its owning task while this exchange can be cancelled. pub(crate) async fn exchange_location_generation( compiler: &mut LocationCompiler, generation: i64, -) -> Result, ControlError> { +) -> Result, ControlError> { compiler .stdin .write_all(format!("{generation}\n").as_bytes()) diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 7fc5eb449e..5a8082353a 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -4,6 +4,7 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::time::Duration; +use chrono::Utc; use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; use sqlbuild_orchestration::platform::models::{ LocationHealth, LocationWorkClient, LocationWorkPoll, @@ -110,13 +111,15 @@ async fn watch_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let Some(published) = + let Some((published, refresh_at)) = compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? else { return Ok(()); }; if published { - watch.retry_at = None; + watch.retry_at = refresh_at.map(|deadline| { + Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default() + }); published_generation = Some(generation); let Some(files) = check_published_location( options, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index c8e6917938..627cf9baa9 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -152,14 +152,18 @@ pub(crate) struct LocationWatch { pub(crate) retry_at: Option, } +/// Private interpreter response: publication metadata and its earliest calendar refresh deadline. +pub(crate) type LocationPublication = ( + sqlbuild_orchestration::snapshots::models::SnapshotRecord, + Option>, +); + /// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. #[derive(Debug)] pub(crate) struct LocationCompileRequest { pub(crate) generation: i64, pub(crate) python_sources: Vec, - pub(crate) reply: tokio::sync::oneshot::Sender< - Result, String>, - >, + pub(crate) reply: tokio::sync::oneshot::Sender, String>>, } /// Only this task owns the isolated interpreter and its private stdio protocol. diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 72e6e9b2db..b9754a724f 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -3,7 +3,8 @@ use uuid::Uuid; use crate::declarations::_helpers::trigger_state::trigger_timing; -use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::declarations::models::{Definitions, PreviousTriggerState, Schedule}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -52,12 +53,44 @@ pub(crate) async fn publish_definitions( (transaction, _) = query(transaction, "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; - for (name, trigger) in definitions.triggers { + for (name, mut trigger) in definitions.triggers { let id = Uuid::new_v5( &snapshot.location_id, &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) .map_err(|error| ControlError::Invalid(error.to_string()))?, ); + let previous; + (transaction, previous) = query(transaction, + "SELECT t.definition_json, s.started_at, s.last_tick_for, s.status_override FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id WHERE s.trigger_id=$1", + vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + if let Some(state) = &state + && let Schedule::Calendar { + times: previous_times, + .. + } = &state.definition_json.schedule + && let Schedule::Calendar { times, .. } = &mut trigger.schedule + { + times.extend( + previous_times + .iter() + .filter(|time| { + **time >= state.started_at + && **time <= snapshot.published_at + && state.last_tick_for.is_none_or(|last| **time > last) + }) + .copied(), + ); + if times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid(format!( + "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" + ))); + } + } let definition = serde_json::to_value(&trigger) .map_err(|error| ControlError::Invalid(error.to_string()))?; (transaction, _) = query(transaction, @@ -65,13 +98,6 @@ pub(crate) async fn publish_definitions( ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; - let previous; - (transaction, previous) = query(transaction, "SELECT started_at, last_tick_for, status_override FROM sqb_trigger_state WHERE trigger_id=$1", vec![Parameter::Id(id)]).await?; - let state = previous - .into_iter() - .next() - .map(decode_row::) - .transpose()?; let started_at = state.as_ref().map_or( trigger.starts_at.unwrap_or(snapshot.published_at), |state| state.started_at, diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 623d581f43..98f516c0d3 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -142,6 +142,7 @@ pub(crate) struct EditableTriggerState { #[derive(Clone, Debug, Deserialize)] pub(crate) struct PreviousTriggerState { + pub(crate) definition_json: TriggerDefinition, pub(crate) started_at: DateTime, pub(crate) last_tick_for: Option>, pub(crate) status_override: Option, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 1dd53ec524..2c05267eb7 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -247,3 +247,126 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ }); } } + +#[test] +fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_survive_and_only_future_times_are_replaced() + { + use crate::runs::main::list_runs::list_runs; + use crate::scheduler::main::list_ticks::list_ticks; + use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; + use serde_json::json; + + for test_case in [ + TickTransactionTestCase { + description: "SQLite retains due calendar ticks", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL retains due calendar ticks", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ] { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish pending calendar"); + input.generation = 2; + let future: chrono::DateTime = + "2031-01-01T00:00:00Z".parse().expect("future"); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = json!( + (0..10_000) + .map(|seconds| future + chrono::TimeDelta::seconds(seconds)) + .collect::>() + ); + assert!( + publish_snapshot(store, &input) + .await + .expect_err("pending ticks cannot be truncated to fit a refreshed horizon") + .to_string() + .contains("let the scheduler drain pending ticks") + ); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2031-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh while scheduler is offline"); + let trigger = list_triggers(store, location, "development") + .await + .expect("refreshed trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + json!([ + "2020-01-01T00:00:00Z", + "2020-01-02T00:00:00Z", + "2020-01-03T00:00:00Z", + "2031-01-01T00:00:00Z" + ]), + "{}", + test_case.description + ); + assert_eq!( + trigger.next_due_at, + Some("2020-01-01T00:00:00Z".parse().expect("due")) + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + let plan = evaluate_trigger(store, &lease, trigger.trigger_id) + .await + .expect("catch up") + .expect("trigger"); + assert_eq!(plan.actions.len(), test_case.expected_ticks); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("ticks") + .len(), + test_case.expected_ticks + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("coalesced run") + .len(), + test_case.expected_runs + ); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2032-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh after catch-up"); + let trigger = list_triggers(store, location, "development") + .await + .expect("drained trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + json!(["2032-01-01T00:00:00Z"]) + ); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("immutable tick history") + .len(), + test_case.expected_ticks + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 270745994d..621a30947c 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -16,6 +16,8 @@ default = [] extension-module = ["pyo3/extension-module"] [dependencies] +chrono.workspace = true +chrono-tz.workspace = true fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 0d95fcf560..bbb369fa17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -15,7 +15,7 @@ use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_re use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - AuthoredDeclaration, compile_declarations, discover_declarations, + compile_declarations, discover_declarations, }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; @@ -126,13 +126,15 @@ pub(crate) fn build_orchestration_snapshot( None => discover_declarations(py, &project_dir)?, }; if !declarations.is_empty() { - let declarations = collect_declarations(py, declarations)?; - let authored = declarations - .iter() - .map(|declaration| declaration.extract::()) - .collect::>>()?; + let authored = collect_declarations(py, declarations)?; let graph = project.lineage_graph(py)?; - input = compile_declarations(input, graph.bind(py).get().graph(), authored)?; + input = compile_declarations( + py, + input, + &project_dir, + graph.bind(py).get().graph(), + authored, + )?; } Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index d5922477fc..7bc75bb723 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -3,7 +3,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; @@ -17,13 +16,16 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - GroupDefinition, JobDefinition, OnFailure, Schedule, TriggerDefinition, + Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDefinition, }; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - authored_setting, module_jobs, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, module_jobs, +}; +use crate::bindings::_helpers::native_host::orchestration_triggers::{ + compile_calendars, compile_cron, }; /// Dedicated declaration modules are imported only in the code-location interpreter. @@ -55,82 +57,10 @@ pub(crate) fn discover_declarations<'py>( Ok(declarations) } -#[derive(FromPyObject)] -struct AuthoredSchedule { - cron: String, - timezone: String, - #[pyo3(from_py_with = authored_setting)] - starts_at: Value, - default_status: String, - catch_up: String, - #[pyo3(from_py_with = authored_setting)] - max_catch_up: Value, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredJob { - name: String, - description: String, - select: String, - exclude: String, - overlap: String, - on_failure: String, - #[pyo3(from_py_with = authored_setting)] - settings: Value, - #[pyo3(from_py_with = authored_setting)] - concurrency: Value, - concurrency_groups: Vec, - schedule: Option, - targets: Vec, -} - -#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] -struct AuthoredJobReference { - name: String, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredGroup { - name: String, - description: String, - jobs: Vec, - after: BTreeMap>, - overlap: String, - concurrency_groups: Vec, - schedule: Option, - targets: Vec, -} - -#[derive(FromPyObject)] -pub(crate) enum AuthoredDeclaration { - Job(AuthoredJob), - Group(AuthoredGroup), -} - -impl AuthoredSchedule { - fn compile(self, name: &str, description: &str) -> PyResult { - let catch_up = if self.max_catch_up.is_null() { - Value::String(self.catch_up) - } else { - serde_json::json!({self.catch_up: {"max_runs": self.max_catch_up}}) - }; - Ok(TriggerDefinition { - description: description.into(), - job_key: name.into(), - schedule: Schedule::Cron { expression: self.cron, timezone: self.timezone }, - starts_at: serde_json::from_value(self.starts_at) - .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, - default_status: serde_json::from_value(Value::String(self.default_status)) - .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, - catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( - "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" - )))?, - }) - } -} - pub(crate) fn compile_declarations( + py: Python<'_>, mut input: SnapshotInput, + project_dir: &Path, graph: &ProjectGraph, declarations: Vec, ) -> PyResult { @@ -148,10 +78,12 @@ pub(crate) fn compile_declarations( let mut jobs: BTreeMap = BTreeMap::new(); let mut authored_groups: Vec = Vec::new(); let mut authored_jobs: Vec = Vec::new(); + let mut authored_calendars = Vec::new(); for declaration in declarations { match declaration { AuthoredDeclaration::Job(job) => authored_jobs.push(job), AuthoredDeclaration::Group(group) => authored_groups.push(group), + AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), } } let mut triggers: BTreeMap = BTreeMap::new(); @@ -193,7 +125,7 @@ pub(crate) fn compile_declarations( if let Some(schedule) = job.schedule { triggers.insert( job.name.clone(), - schedule.compile(&job.name, &job.description)?, + compile_cron(schedule.bind(py), &job.name, &job.description)?, ); } jobs.insert( @@ -244,7 +176,7 @@ pub(crate) fn compile_declarations( if let Some(schedule) = group.schedule { triggers.insert( group.name.clone(), - schedule.compile(&group.name, &group.description)?, + compile_cron(schedule.bind(py), &group.name, &group.description)?, ); } groups.insert( @@ -263,6 +195,20 @@ pub(crate) fn compile_declarations( }, ); } + let Definitions { + jobs, + groups, + triggers, + } = compile_calendars( + py, + (project_dir, &input.target), + authored_calendars, + Definitions { + jobs, + groups, + triggers, + }, + )?; for (key, document) in [ ("jobs", serde_json::to_value(jobs)), ("groups", serde_json::to_value(groups)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index fdd7526b6b..6f9e92f4f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -1,13 +1,16 @@ //! Compile in the location interpreter and publish data through its scoped HTTP credential. +use std::collections::BTreeMap; use std::path::Path; use std::process::{Command, Stdio}; +use chrono::{DateTime, Utc}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_orchestration::declarations::models::{Schedule, TriggerDefinition}; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; @@ -129,7 +132,12 @@ fn compile_location<'py>( view, &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, project_dir.extract()?, - None, + // Workers verify nodes and environment, not dynamic calendar declarations. + arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action == "execute") + .then(Vec::new), )?; status(py, "Location snapshot compiled.")?; Ok((snapshot, discovered)) @@ -159,7 +167,7 @@ pub(super) fn location_arguments<'py>( pub(super) fn publish_location( py: Python<'_>, arguments: &KeywordArguments<'_>, -) -> PyResult { +) -> PyResult<(SnapshotRecord, Option>)> { if arguments .value("action")? .extract::>()? @@ -182,6 +190,22 @@ pub(super) fn publish_location( .extract::()?; let input = metadata(py, arguments, &directory)?; let (snapshot, _) = compile_location(py, arguments, &input)?; + let triggers: BTreeMap = serde_json::from_value( + snapshot + .input + .documents + .get("triggers") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let refresh_at = triggers + .values() + .filter_map(|trigger| match &trigger.schedule { + Schedule::Calendar { refresh_at, .. } => Some(*refresh_at), + Schedule::Cron { .. } => None, + }) + .min(); status(py, "Publishing location snapshot...")?; let record = snapshot.publication( py, @@ -196,7 +220,7 @@ pub(super) fn publish_location( PyValueError::new_err(message) })?; status(py, "Location snapshot published.")?; - Ok(record) + Ok((record, refresh_at)) } /// One-shot publication and the supervisor's private stdio runtime share the same compiler path. @@ -249,7 +273,7 @@ pub(crate) fn run_location<'py>( return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); } let publication = (|| { - let record = publish_location(py, &arguments)?; + let (record, _) = publish_location(py, &arguments)?; let output = serde_json::to_string(&record) .map_err(|error| PyValueError::new_err(error.to_string()))?; write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 25adcc06b4..e604e71354 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod host; pub(crate) mod model_lookup; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; +pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index d758b406a0..1b04b08ad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,15 +1,57 @@ //! The public authoring records enter native compilation here, never scheduler execution. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::path::Path; +use pyo3::FromPyObject; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use serde_json::Value; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::native_host::host::entry; +#[derive(FromPyObject)] +pub(crate) struct AuthoredJob { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) select: String, + pub(crate) exclude: String, + pub(crate) overlap: String, + pub(crate) on_failure: String, + #[pyo3(from_py_with = authored_setting)] + pub(crate) settings: Value, + #[pyo3(from_py_with = authored_setting)] + pub(crate) concurrency: Value, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct AuthoredJobReference { + pub(crate) name: String, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredGroup { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) jobs: Vec, + pub(crate) after: BTreeMap>, + pub(crate) overlap: String, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +pub(crate) enum AuthoredDeclaration { + Job(AuthoredJob), + Group(AuthoredGroup), + Calendar(Py), +} + /// JSON authoring values cross once; the interpreter rejects circular user containers safely. pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { let text: String = value @@ -32,6 +74,9 @@ pub(crate) fn module_jobs<'py>( let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; let group_class = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let calendar_class = py + .import("sqlbuild.orchestration")? + .getattr("CalendarSchedule")?; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); @@ -40,7 +85,10 @@ pub(crate) fn module_jobs<'py>( if !seen.insert(value.as_ptr() as usize) { continue; } - if value.is_instance(&job_class)? || value.is_instance(&group_class)? { + if value.is_instance(&job_class)? + || value.is_instance(&group_class)? + || value.is_instance(&calendar_class)? + { jobs.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { pending.extend(value.try_iter()?.collect::>>()?); @@ -52,12 +100,18 @@ pub(crate) fn module_jobs<'py>( pub(crate) fn collect_declarations<'py>( py: Python<'py>, declarations: Vec>, -) -> PyResult>> { +) -> PyResult> { let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; let group = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let calendar = py + .import("sqlbuild.orchestration")? + .getattr("CalendarSchedule")?; let mut roots: HashSet = HashSet::new(); for declaration in &declarations { if !roots.insert(declaration.as_ptr() as usize) { + if declaration.is_instance(&calendar)? { + return Err(PyValueError::new_err("duplicate calendar declaration")); + } return Err(PyValueError::new_err(format!( "duplicate job {}", declaration.getattr("name")?.extract::()? @@ -66,12 +120,21 @@ pub(crate) fn collect_declarations<'py>( } let mut pending = declarations; let mut seen: HashSet = HashSet::new(); - let mut collected: Vec> = Vec::new(); + let mut collected: Vec = Vec::new(); while let Some(declaration) = pending.pop() { if !seen.insert(declaration.as_ptr() as usize) { continue; } - if declaration.is_instance(&group)? { + let authored = if declaration.is_instance(&calendar)? { + let target = declaration.getattr("job")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "calendar job must be a Job or JobGroup object", + )); + } + pending.push(target); + AuthoredDeclaration::Calendar(declaration.unbind()) + } else if declaration.is_instance(&group)? { for member in declaration.getattr("jobs")?.try_iter()? { let member = member?; if !member.is_instance(&job)? { @@ -93,12 +156,15 @@ pub(crate) fn collect_declarations<'py>( } } } + AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( - "snapshot declarations must be sqlbuild.orchestration.Job or JobGroup objects", + "snapshot declarations must be Job, JobGroup or CalendarSchedule objects", )); - } - collected.push(declaration); + } else { + AuthoredDeclaration::Job(declaration.extract()?) + }; + collected.push(authored); } Ok(collected) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs new file mode 100644 index 0000000000..198541d498 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -0,0 +1,243 @@ +//! Authored trigger controls and bounded calendars evaluated only in the location interpreter. + +use std::collections::BTreeSet; +use std::path::Path; + +use chrono::{DateTime, TimeDelta, Timelike, Utc}; +use chrono_tz::Tz; +use pyo3::FromPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_declarations::authored_setting; + +#[derive(FromPyObject)] +struct AuthoredControls { + #[pyo3(from_py_with = authored_setting)] + starts_at: Value, + default_status: String, + catch_up: String, + #[pyo3(from_py_with = authored_setting)] + max_catch_up: Value, +} + +fn compile_trigger( + authored: &Bound<'_, PyAny>, + job_key: &str, + description: &str, + schedule: Schedule, +) -> PyResult { + let controls = authored.extract::()?; + let catch_up = if controls.max_catch_up.is_null() { + Value::String(controls.catch_up) + } else { + serde_json::json!({controls.catch_up: {"max_runs": controls.max_catch_up}}) + }; + Ok(TriggerDefinition { + description: description.into(), + job_key: job_key.into(), + schedule, + starts_at: serde_json::from_value(controls.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(controls.default_status)) + .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, + }) +} + +pub(crate) fn compile_cron( + authored: &Bound<'_, PyAny>, + name: &str, + description: &str, +) -> PyResult { + compile_trigger( + authored, + name, + description, + Schedule::Cron { + expression: authored.getattr("cron")?.extract()?, + timezone: authored.getattr("timezone")?.extract()?, + }, + ) +} + +fn horizon(value: &str) -> PyResult { + let invalid = || { + PyValueError::new_err( + "calendar horizon must be a positive fixed duration using d, h, m or s, within datetime range", + ) + }; + let digits = parse_duration(value).ok_or_else(invalid)?; + if [&digits.years, &digits.months] + .iter() + .any(|digits| !digits.trim_start_matches('0').is_empty()) + { + return Err(invalid()); + } + let seconds = [ + (digits.days, 86_400_i64), + (digits.hours, 3_600), + (digits.minutes, 60), + (digits.seconds, 1), + ] + .into_iter() + .try_fold(0_i64, |total, (digits, factor)| { + total.checked_add(digits.parse::().ok()?.checked_mul(factor)?) + }) + .ok_or_else(invalid)?; + TimeDelta::try_seconds(seconds).ok_or_else(invalid) +} + +fn calendar_times( + authored: &Bound<'_, PyAny>, + session: &Bound<'_, PyAny>, + providers: &Bound<'_, PyDict>, + start: DateTime, +) -> PyResult { + let py = authored.py(); + let timezone: String = authored.getattr("timezone")?.extract()?; + timezone + .parse::() + .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; + let duration = horizon(&authored.getattr("horizon")?.extract::()?)?; + let end = start + .checked_add_signed(duration) + .ok_or_else(|| PyValueError::new_err("calendar horizon is out of datetime range"))?; + let datetime = imported(py, "datetime", "datetime")?; + let zone = imported(py, "zoneinfo", "ZoneInfo")?.call1((timezone,))?; + let window = PyDict::new(py); + for (name, instant) in [("start", start), ("end", end)] { + window.set_item( + name, + datetime + .call_method1("fromisoformat", (instant.to_rfc3339(),))? + .call_method1("astimezone", (&zone,))?, + )?; + } + let window = py + .import("sqlbuild.orchestration")? + .getattr("CalendarWindow")? + .call((), Some(&window))?; + let function = authored.getattr("function")?; + let arguments = PyDict::new(py); + arguments.set_item("window", window)?; + for usage in provider_usages(&function, providers)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? == "window" { + continue; + } + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + let mut times = BTreeSet::new(); + for (index, time) in function + .call((), Some(&arguments))? + .try_iter()? + .take(MAX_TICKS_PER_BATCH + 1) + .enumerate() + { + if index == MAX_TICKS_PER_BATCH { + return Err(PyValueError::new_err(format!( + "calendar exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + let time = time?; + if !time.is_instance(&datetime)? { + return Err(PyValueError::new_err( + "calendar times must be timezone-aware datetimes", + )); + } + let time = + DateTime::parse_from_rfc3339(&time.call_method0("isoformat")?.extract::()?) + .map_err(|_| { + PyValueError::new_err("calendar times must be timezone-aware datetimes") + })? + .with_timezone(&Utc); + if time < start || time >= end { + return Err(PyValueError::new_err( + "calendar time lies outside its half-open evaluation window", + )); + } + times.insert(time); + } + Ok(Schedule::Calendar { + times, + refresh_at: start + duration / 2, + }) +} + +pub(crate) fn compile_calendars( + py: Python<'_>, + (project_dir, target): (&Path, &str), + authored: Vec>, + mut definitions: Definitions, +) -> PyResult { + let mut calendars = Vec::new(); + for calendar in authored { + let targets = calendar + .bind(py) + .getattr("targets")? + .extract::>()?; + if targets.is_empty() || targets.iter().any(|value| value == target) { + calendars.push(calendar); + } + } + if calendars.is_empty() { + return Ok(definitions); + } + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + let now = Utc::now(); + let start = now + .with_nanosecond(now.timestamp_subsec_micros() * 1_000) + .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; + within(&session, |_| { + for calendar in calendars { + let calendar = calendar.bind(py); + let name: String = calendar + .getattr("function")? + .getattr("__name__")? + .extract()?; + let job_key: String = calendar.getattr("job")?.getattr("name")?.extract()?; + let description: String = calendar.getattr("description")?.extract()?; + if description.trim().is_empty() + || (!definitions.jobs.contains_key(&job_key) + && !definitions.groups.contains_key(&job_key)) + { + return Err(PyValueError::new_err(format!( + "calendar {name} needs a description and a published job" + ))); + } + if definitions.triggers.contains_key(&name) { + return Err(PyValueError::new_err(format!("duplicate trigger {name}"))); + } + let schedule = calendar_times(calendar, &session, &by_name, start)?; + let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; + trigger.starts_at.get_or_insert(start); + definitions.triggers.insert(name, trigger); + } + Ok(definitions) + }) +} diff --git a/fensu.toml b/fensu.toml index 104f5bec85..67c1d437d9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2661,6 +2661,7 @@ module_layers = [ "model_config=front", "native_host=infra", "native_host/model_lookup=project", + "native_host/orchestration_triggers=project", "native_threads=infra", "output_capture=output", "plan_command=commands", diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index d305deef89..997e6d0181 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,9 +1,27 @@ """Public authoring and snapshot compilation API for orchestration declarations.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models import Job, JobGroup, Schedule +from sqlbuild.orchestration_declarations.models import ( + CalendarSchedule, + CalendarWindow, + Job, + JobGroup, + Schedule, +) +from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group from sqlbuild.orchestration_declarations.models import Schedule as schedule -__all__ = ("Job", "JobGroup", "Schedule", "compile_snapshot", "job", "job_group", "schedule") +__all__ = ( + "CalendarSchedule", + "CalendarWindow", + "Job", + "JobGroup", + "Schedule", + "calendar_schedule", + "compile_snapshot", + "job", + "job_group", + "schedule", +) diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index cc10a5eb4a..b9e09facb3 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,7 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import Job, JobGroup +from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup def compile_snapshot( @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job | JobGroup] | None = None, + declarations: Sequence[Job | JobGroup | CalendarSchedule] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index e113ce4aa4..b9617e4ffe 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -2,8 +2,9 @@ from __future__ import annotations -from collections.abc import Sequence -from dataclasses import dataclass, field +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime from typing import Literal @@ -50,3 +51,30 @@ class JobGroup: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, kw_only=True) +class CalendarWindow: + """Timezone-aware, half-open interval for an upcoming calendar evaluation.""" + + start: datetime + end: datetime + + +@dataclass(frozen=True, kw_only=True, eq=False) +class CalendarSchedule: + """Decorate a location callback returning upcoming timezone-aware datetimes.""" + + job: Job | JobGroup + description: str + horizon: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + catch_up: Literal["latest", "all"] = "latest" + max_catch_up: int | None = None + targets: tuple[str, ...] = () + function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + + def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + return replace(self, function=function) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index e6b239fffe..67d5f94e18 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -127,3 +127,18 @@ class BackfillExecutionTestCase(WorkerExecutionTestCase): description: str window_arguments: tuple[str, ...] = () expected_window: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CalendarRejectionTestCase: + description: str + expression: str + expected_message: str + horizon: str = "8s" + timezone: str = "UTC" + + +@dataclass(frozen=True) +class CalendarRefreshTestCase: + description: str + expected_timezone: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py new file mode 100644 index 0000000000..f15bea15b5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -0,0 +1,285 @@ +"""Calendar callbacks remain in locations; native publication and refresh preserve good state.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.error import HTTPError + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CalendarRefreshTestCase, + CalendarRejectionTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + scheduled_worker, + snapshot_api, + wait_for_generation, + wait_for_location_health, + wait_for_run, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRejectionTestCase( + "naive time", "[datetime(2026, 1, 1)]", "timezone-aware datetimes" + ), + CalendarRejectionTestCase("exclusive end", "[window.end]", "half-open evaluation window"), + CalendarRejectionTestCase( + "before start", + "[window.start - timedelta(microseconds=1)]", + "half-open evaluation window", + ), + CalendarRejectionTestCase( + "non-datetime", "['2026-01-01T00:00:00Z']", "timezone-aware datetimes" + ), + CalendarRejectionTestCase( + "unbounded generator", "repeat(window.start)", "exceeds 10000 upcoming times" + ), + CalendarRejectionTestCase("zero horizon", "[]", "positive fixed duration", horizon="0s"), + CalendarRejectionTestCase( + "overflowing horizon", + "[]", + "positive fixed duration", + horizon="999999999999999999999999999d", + ), + CalendarRejectionTestCase("unknown zone", "[]", "IANA timezone", timezone="Unknown/Orders"), + ], + ids=lambda case: case.description, +) +def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_validation_is_bounded_and_no_snapshot_is_committed( + test_case: CalendarRejectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": ( + "from datetime import datetime, timedelta\nfrom itertools import repeat\n" + "from sqlbuild.orchestration import calendar_schedule, job\n" + "orders = job('orders', description='Order reporting.', select='orders')\n" + f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" + f"def deliveries(window):\n return {test_case.expression}\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + "--generation", + "1", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_message in result.stderr + with pytest.raises(HTTPError) as failure: + current_snapshot(api, "development") + assert failure.value.code == 404 + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRefreshTestCase("offset-aware calendar refresh", "America/New_York"), + CalendarRefreshTestCase("UTC calendar refresh", "UTC"), + ], + ids=lambda case: case.description, +) +def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refresh_reuses_the_interpreter_and_keeps_last_good( + test_case: CalendarRefreshTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + data: Path = tmp_path / "fulfillment.json" + evaluations: Path = tmp_path / "evaluations.jsonl" + data.write_text(json.dumps({"offsets": [3, 0, 3]})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_CALENDAR_EVALUATIONS", str(evaluations)) + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py" + ) + .read_text() + .replace("America/New_York", test_case.expected_timezone) + + ( + "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" + "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" + ), + "providers/fulfillment.py": ( + "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Upcoming synthetic fulfillment times."""\n' + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_CALENDAR_EVALUATIONS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def upcoming(self, window):\n" + " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" + " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" + " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" + " return [window.start + timedelta(seconds=offset) for offset in data['offsets']]\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, publications) as process: + first: dict[str, Any] = wait_for_generation(api, "development", 1, process, log) + second: dict[str, Any] = wait_for_generation(api, "development", 2, process, log) + assert first["nodes"] == second["nodes"] + assert second["changes"] == [] + assert first["documents"]["triggers"].keys() == {"fulfillment_calendar"} + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: + stored, next_due = control.execute( + "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id)" + ).fetchone() + original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ + "schedule" + ]["times"] + assert json.loads(stored)["schedule"]["times"] == sorted( + original_times + + second["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"] + ) + assert datetime.fromisoformat(next_due) == datetime.fromisoformat(original_times[0]) + observations: list[dict[str, Any]] = [ + json.loads(line) for line in evaluations.read_text().splitlines() + ] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 2 + successful: list[dict[str, Any]] = observations[1::3] + assert len(successful) == 2 + assert len({row["pid"] for row in observations}) == 1 + assert all(row["zone"] == test_case.expected_timezone for row in successful) + for snapshot, evaluation in zip([first, second], successful, strict=True): + start: datetime = datetime.fromisoformat(evaluation["start"]) + end: datetime = datetime.fromisoformat(evaluation["end"]) + assert end - start == timedelta(seconds=8) + schedule: dict[str, Any] = snapshot["documents"]["triggers"][ + "fulfillment_calendar" + ]["schedule"] + assert schedule["kind"] == "calendar" + assert [datetime.fromisoformat(value) for value in schedule["times"]] == [ + start + timedelta(seconds=offset) for offset in [0, 3] + ] + assert datetime.fromisoformat(schedule["refresh_at"]) == start + timedelta( + seconds=4 + ) + data.write_text(json.dumps({"fail": True})) + _ = wait_for_location_health( + api, "development", {"kind": "compile_failed", "generation": 3}, process, log + ) + assert current_snapshot(api, "development")["record"] == second["record"] + observations = [json.loads(line) for line in evaluations.read_text().splitlines()] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 3 + data.write_text(json.dumps({"offsets": [1]})) + (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(19 AS INTEGER)" + ) + ) + third: dict[str, Any] = wait_for_generation(api, "development", 3, process, log) + assert "CAST(19 AS INTEGER)" in third["nodes"]["model:stg_orders"]["raw_sql"] + assert ( + len(third["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"]) + == 1 + ) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="calendar evaluation belongs only to the location", + model_sql="SELECT 29 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(29,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not_evaluate_the_calendar_again( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + evaluated: Path = tmp_path / "evaluated" + test_case = replace( + test_case, + files=test_case.files + | { + "orchestration/orders.py": ( + "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" + "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" + "def deliveries(window):\n" + f" marker = Path({str(evaluated)!r})\n" + " if marker.exists():\n raise ValueError('calendar evaluated inside a run worker')\n" + " marker.write_text('evaluated')\n return []\n" + ) + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert evaluated.read_text() == "evaluated" + completed: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows From e69a97679638c7a5704d12637281dbcaddb80892 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:34:28 +0000 Subject: [PATCH 043/138] test: avoid writable native executable copies --- crates/sqlbuild-cli/tests/native_commands/helpers.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 606246903b..ec2421c381 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -6,8 +6,10 @@ use crate::test_types::{ServiceTestCase, TestControl}; pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { let binary = directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX)); - std::fs::copy(env!("CARGO_BIN_EXE_sqb"), &binary) - .expect("copy standalone binary without Python CLI"); + // Do not open the executable for writing: concurrent process launches can inherit + // the copy's writable descriptor until exec, making this launch fail with ETXTBSY. + std::fs::hard_link(env!("CARGO_BIN_EXE_sqb"), &binary) + .expect("link standalone binary without Python CLI"); binary } From dde985ce93a3bad9f8a9b138dc16ca2fa580a844 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:00:28 +0000 Subject: [PATCH 044/138] fix: distinguish PyO3 metadata from Python crossings --- fensu.toml | 204 +++++++++--------- pyproject.toml | 2 +- .../fensu_policy/_helpers/native_crossings.py | 8 +- .../scripts/fensu_policy/__init__.py | 1 + .../scripts/fensu_policy/_test_types.py | 10 + .../fensu_policy/test_native_boundary.py | 78 +++++++ .../rules/test_native_boundary.py | 31 +++ uv.lock | 30 +-- 8 files changed, 239 insertions(+), 125 deletions(-) create mode 100644 tests/integration/scripts/fensu_policy/__init__.py create mode 100644 tests/integration/scripts/fensu_policy/_test_types.py create mode 100644 tests/integration/scripts/fensu_policy/test_native_boundary.py diff --git a/fensu.toml b/fensu.toml index 67c1d437d9..aefc72b446 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1570,18 +1570,6 @@ path = "crates/sqlbuild-orchestration/src/credentials/models.rs" symbols = ["IssuedToken"] reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." -[[targets.rust.rule_exceptions]] -rule = "XSB072" -path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs" -symbols = ["NativeOrchestrationSnapshot"] -reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only SnapshotInput; binding functions remain checked." - -[[targets.rust.rule_exceptions]] -rule = "XSB072" -path = "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs" -symbols = ["NativeOrchestrationWorker"] -reason = "The pyclass module attribute names sqlbuild._native but executes no Python call. The struct retains only native worker ownership; binding functions remain checked." - [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" @@ -1687,17 +1675,17 @@ edge_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/records/registry.rs", ] pending_budgets = [ - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1712,43 +1700,43 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", @@ -1757,19 +1745,19 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=42", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs=45", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=40", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=38", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs=39", "crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=3", @@ -1797,22 +1785,22 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=8", @@ -1829,8 +1817,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", @@ -1852,11 +1840,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs=19", @@ -1868,8 +1856,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", @@ -1878,7 +1866,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", @@ -1894,7 +1882,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs=6", @@ -1909,16 +1897,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=29", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=16", @@ -1943,7 +1931,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs=5", @@ -1954,7 +1942,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs=4", @@ -1982,7 +1970,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", @@ -1992,11 +1980,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", @@ -2007,16 +1995,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", @@ -2036,10 +2024,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", @@ -2059,7 +2047,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", @@ -2073,9 +2061,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", @@ -2108,11 +2096,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=1", @@ -2120,12 +2108,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=1", @@ -2137,8 +2125,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", @@ -2149,8 +2137,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=1", @@ -2165,24 +2153,24 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=20", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=14", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=1", @@ -2190,11 +2178,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=21", - "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs=8", @@ -2209,7 +2197,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=1", @@ -2221,26 +2209,26 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=32", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", @@ -2248,7 +2236,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=101", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=100", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", @@ -2261,8 +2249,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs=4", @@ -2275,7 +2263,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=8", - "crates/sqlbuild-python/src/bindings/models.rs=7", + "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB073] diff --git a/pyproject.toml b/pyproject.toml index f66e346fcf..c29137b916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.0", + "fensu>=0.28.1", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/scripts/fensu_policy/_helpers/native_crossings.py b/scripts/fensu_policy/_helpers/native_crossings.py index 430cc3fb78..27df706697 100644 --- a/scripts/fensu_policy/_helpers/native_crossings.py +++ b/scripts/fensu_policy/_helpers/native_crossings.py @@ -27,6 +27,11 @@ _PYTHON_FIELD: re.Pattern[str] = re.compile(NATIVE_PYTHON_FIELD_PATTERN) _STRUCT_OPEN: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?struct\s+\w+[^;]*\{\s*$") _FIELD: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?\w+\s*:") +_PYCLASS_MODULE: re.Pattern[str] = re.compile( + r"(^[ \t]*#\[\s*(?:pyo3::)?pyclass\s*\([^]]*?\bmodule\s*=\s*(?:r#*)?)" + r'"(?:\\.|[^"\\\n])*"', + re.MULTILINE, +) def is_checked_binding(*, path: str) -> bool: @@ -72,8 +77,9 @@ def _code_lines(*, source: str) -> list[tuple[int, str]]: def crossing_lines(*, source: str) -> list[int]: - """Line numbers of Rust-to-Python crossings, one entry per crossing.""" + """Line numbers of Rust-to-Python crossings, excluding inert PyO3 module metadata.""" + source = _PYCLASS_MODULE.sub(r'\1""', source) found: list[int] = [] number: int line: str diff --git a/tests/integration/scripts/fensu_policy/__init__.py b/tests/integration/scripts/fensu_policy/__init__.py new file mode 100644 index 0000000000..6aab21b701 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/__init__.py @@ -0,0 +1 @@ +"""Integration checks for repository-authored Fensu policies.""" diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py new file mode 100644 index 0000000000..ef287d1b65 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -0,0 +1,10 @@ +"""Native boundary metadata cases for the real Fensu CLI.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeBoundaryTestCase: + description: str + metadata: str + expected_fault_count: int diff --git a/tests/integration/scripts/fensu_policy/test_native_boundary.py b/tests/integration/scripts/fensu_policy/test_native_boundary.py new file mode 100644 index 0000000000..1b996fae04 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_native_boundary.py @@ -0,0 +1,78 @@ +"""Exercise native boundary policy through Fensu's real Rust analyzer and rule host.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import NativeBoundaryTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + NativeBoundaryTestCase( + description="pyclass module metadata", + metadata='#[pyclass(frozen, module = "sqlbuild._native")]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + NativeBoundaryTestCase( + description="namespaced pyclass raw module metadata", + metadata='#[pyo3::pyclass(frozen, module = r#"sqlbuild._native"#)]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_pyclass_metadata_when_checking_rust_then_only_real_python_crossings_fault( + tmp_path: Path, + test_case: NativeBoundaryTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + tmp_path / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (tmp_path / "scripts/__init__.py").write_text("") + (tmp_path / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = tmp_path / "crates/sqlbuild-python" + (crate / "src").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + (tmp_path / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.native_boundary"]\n' + 'select = ["XSB072"]\n' + ) + source: Path = crate / "src/lib.rs" + source.write_text(test_case.metadata) + command: list[str] = [ + str(Path(sys.executable).with_name("fensu")), + "check", + "--target", + "rust", + "--no-cache", + "--no-color", + ] + clean: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "Found 0 faults" in clean.stdout + source.write_text( + test_case.metadata + 'fn read(py: Python<\'_>) { py.import("sqlbuild.compiler"); }\n' + ) + crossing: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert crossing.returncode == 1, crossing.stdout + crossing.stderr + assert crossing.stdout.count("XSB072") == test_case.expected_fault_count, crossing.stdout + assert crossing.stdout.count(" --> crates/sqlbuild-python/src/lib.rs:3:0") == 2 + assert " --> crates/sqlbuild-python/src/lib.rs:1:0" not in crossing.stdout + assert source.read_text().startswith(test_case.metadata) diff --git a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py index dd3e4ba20f..9422bcd69a 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py +++ b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py @@ -135,6 +135,37 @@ def test_given_test_module_when_checking_native_patches_then_matches_contract( ), expected_lines=(3, 4, 5, 5, 6, 6), ), + NativeLinesTestCase( + description="pyclass module metadata is not a Python crossing", + source='#[pyclass(frozen, module = "sqlbuild._native")]\nstruct Snapshot;\n', + expected_lines=(), + ), + NativeLinesTestCase( + description="multiline namespaced pyclass metadata preserves callback line numbers", + source=dedent( + """ + #[pyo3::pyclass( + frozen, + module = "sqlbuild._native", + )] + struct Snapshot; + fn read(py: Python<'_>) { + py.import("sqlbuild.compiler"); + } + """ + ), + expected_lines=(8, 8), + ), + NativeLinesTestCase( + description="raw pyclass module metadata does not hide an adjacent real crossing", + source='#[pyclass(module = r#"sqlbuild._native"#)] struct Snapshot; fn read() { imported(py, "sqlbuild.compiler", "compile"); }\n', + expected_lines=(1, 1), + ), + NativeLinesTestCase( + description="module assignments outside pyclass remain crossings", + source='fn read() { let module = "sqlbuild.compiler"; load(module); }\n', + expected_lines=(1,), + ), NativeLinesTestCase( description="comments, imports and test modules are skipped", source=dedent( diff --git a/uv.lock b/uv.lock index 7f698d1833..17225a2936 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.0" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/c7/baf72530c3a67a6d26957658475d32e11adbeb2dbb452425fd5d4af1f575/fensu-0.28.1.tar.gz", hash = "sha256:b81119f0301580367799b77312d5fd82f3ce5a14613b5b709537bca4c070b912", size = 539035, upload-time = "2026-10-31T19:33:41.927Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, - { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, - { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/e8/02/8288dc1ecb853941750d25c2b1effc6fe68d9eeeb188f3d289823731cf69/fensu-0.28.1-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bf1d49d2c800e7c8acdba031c12e6ea1a38544f45c0b75fbe8c35369b63ee98b", size = 4154268, upload-time = "2026-10-31T19:33:24.239Z" }, + { url = "https://files.pythonhosted.org/packages/4c/70/f21686ef9b89f70c84ea3a94dcdbda47839263496c36aa962c15c2eb093e/fensu-0.28.1-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:7e7564b2c4ffb9a031659b3bd82368be70deb97c3f4f620ad05ddfb5f8086e9c", size = 4004303, upload-time = "2026-10-31T19:33:25.807Z" }, + { url = "https://files.pythonhosted.org/packages/3c/50/558ac8fadf943aad5a8a5ba500a3f63e0f5245fd1f66480b6cd8590ce599/fensu-0.28.1-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70da92dad5e5dfbe927bfe870893fe8dc612f28e72780de32014ca42703ba7db", size = 4230255, upload-time = "2026-10-31T19:33:27.649Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5f/b7659a2d6489293a6c16cafc5f8c500b5d5de22536767f468147bed813e3/fensu-0.28.1-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:207a911faee0e14f40005060281ea624b6360f7f35cf6440a02dd0870c4f0bfc", size = 4307330, upload-time = "2026-10-31T19:33:29.361Z" }, + { url = "https://files.pythonhosted.org/packages/69/a6/aa98ad24bb27e3c829c9675c2b753f5d793fb5f546e8b266ade494d6e044/fensu-0.28.1-cp312-abi3-win_amd64.whl", hash = "sha256:88089637c5a52a2e074dd7b29fb76d4e35d1d9d28e8ef4398be30c3bed7d8071", size = 3977891, upload-time = "2026-10-31T19:33:30.84Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.0" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/da/a4ef081ab0ec23cb719dfbce2747c11ee3832a2dda2aaef1f7722548d1e6/fensu_cli-0.28.1.tar.gz", hash = "sha256:4857b2ef850d35bae19462cd6e26fdc0b33c00d97f13f20bb5b00b5c879ef2e9", size = 765839, upload-time = "2026-10-31T19:33:43.051Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, + { url = "https://files.pythonhosted.org/packages/33/03/1c5a56b3d233360d1fb699ff7a92cee005faf8f8fda8c92a7688cc0be167/fensu_cli-0.28.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:67b0bc97cdaeb08e633e68336b1d82b3e598d615b54b9b6f840772929cf6929e", size = 7975688, upload-time = "2026-10-31T19:33:32.569Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a7/d90a388b17a89d9e34238e16e03ce306331c212d215f02ff4c43d6d93189/fensu_cli-0.28.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6a9d97525aea16e079b773b7141a1167bd799bc07ff8bfc720b8f311c91d20b", size = 7492140, upload-time = "2026-10-31T19:33:34.634Z" }, + { url = "https://files.pythonhosted.org/packages/62/f3/4a5247e312d4725ead4ab37d0076371b731a8858c5819cfcceb1d6ce0b69/fensu_cli-0.28.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b8fd61d84ffd2be3c023335e374cabc72eac74dfe5cd90103bde52eba6c40da5", size = 7860157, upload-time = "2026-10-31T19:33:36.407Z" }, + { url = "https://files.pythonhosted.org/packages/d4/71/a954282a9f3f372fe77c942ff6acc8cc2f797b7da82b3b40a6c69715b668/fensu_cli-0.28.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1115a28579b917d6d8c0010e99488318e4cd264236642b5081bb79e8378bdc15", size = 8323299, upload-time = "2026-10-31T19:33:38.357Z" }, + { url = "https://files.pythonhosted.org/packages/35/cb/aa860d0afcb17a7931ab14dcab061d234143caef12e2791d17f27d0816c3/fensu_cli-0.28.1-py3-none-win_amd64.whl", hash = "sha256:4294965dcf4c317b66e4e489c9b9b38e8f8d82000c59bf9f3275fe887a6b45d8", size = 7705331, upload-time = "2026-10-31T19:33:40.314Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.0" }, + { name = "fensu", specifier = ">=0.28.1" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From 21a9a56e5cb0047219983713b2269d355c6b1d06 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:00:34 +0000 Subject: [PATCH 045/138] refactor: align calendar backend ownership boundaries --- .../src/native_commands/_helpers/execution.rs | 5 +- .../sqlbuild-cli/src/native_commands/mod.rs | 1 + .../src/native_commands/models.rs | 7 +- .../sqlbuild-cli/src/native_commands/types.rs | 7 ++ .../declarations/_helpers/trigger_state.rs | 39 ++++++- .../declarations/main/publish_definitions.rs | 106 ++++++++---------- .../tests/scheduler/ticks/test_triggers.rs | 8 +- .../_helpers/compiled_project/snapshot.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 7 +- .../_helpers/location_command/command.rs | 7 +- .../_helpers/location_command/constants.rs | 3 + .../_helpers/native_host/constants.rs | 4 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_triggers.rs | 16 +-- 14 files changed, 124 insertions(+), 90 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index ee24726afd..dac117b46c 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -45,9 +45,10 @@ use crate::native_commands::constants::{ use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationPublication, - LocationServe, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, + LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +use crate::native_commands::types::LocationPublication; use crate::workers::main::open_workers::open_workers; use crate::workers::models::Workers; diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs index d99ca200f2..5ac20489d5 100644 --- a/crates/sqlbuild-cli/src/native_commands/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub(crate) mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 627cf9baa9..b16504cefa 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use uuid::Uuid; use crate::argv::models::Namespace; +use crate::native_commands::types::LocationPublication; /// Privileged local operations require direct ownership of the control database. pub(crate) enum ControlAdmin { @@ -152,12 +153,6 @@ pub(crate) struct LocationWatch { pub(crate) retry_at: Option, } -/// Private interpreter response: publication metadata and its earliest calendar refresh deadline. -pub(crate) type LocationPublication = ( - sqlbuild_orchestration::snapshots::models::SnapshotRecord, - Option>, -); - /// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. #[derive(Debug)] pub(crate) struct LocationCompileRequest { diff --git a/crates/sqlbuild-cli/src/native_commands/types.rs b/crates/sqlbuild-cli/src/native_commands/types.rs new file mode 100644 index 0000000000..438f742268 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/types.rs @@ -0,0 +1,7 @@ +//! Private interpreter protocol shapes owned by the location supervisor. + +use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; + +/// Publication metadata and the earliest calendar refresh deadline. +pub(crate) type LocationPublication = (SnapshotRecord, Option>); diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs index fccc6753f0..b258882455 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -2,10 +2,47 @@ use chrono::{DateTime, TimeDelta, Utc}; -use crate::declarations::models::{TriggerDefinition, TriggerStatus}; +use crate::declarations::models::{ + PreviousTriggerState, Schedule, TriggerDefinition, TriggerStatus, +}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; +/// Replacing a future horizon must retain due ticks not yet consumed by the scheduler. +pub(crate) fn retain_calendar_ticks( + name: &str, + trigger: &mut TriggerDefinition, + state: Option<&PreviousTriggerState>, + snapshot: &SnapshotRecord, +) -> Result<(), ControlError> { + if let Some(state) = state + && let Schedule::Calendar { + times: previous_times, + .. + } = &state.definition_json.schedule + && let Schedule::Calendar { times, .. } = &mut trigger.schedule + { + times.extend( + previous_times + .iter() + .filter(|time| { + **time >= state.started_at + && **time <= snapshot.published_at + && state.last_tick_for.is_none_or(|last| **time > last) + }) + .copied(), + ); + if times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid(format!( + "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" + ))); + } + } + Ok(()) +} + pub(crate) fn trigger_timing( definition: &TriggerDefinition, started_at: DateTime, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index b9754a724f..9dd685ac6c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -2,9 +2,8 @@ use uuid::Uuid; -use crate::declarations::_helpers::trigger_state::trigger_timing; -use crate::declarations::models::{Definitions, PreviousTriggerState, Schedule}; -use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; +use crate::declarations::models::{Definitions, PreviousTriggerState}; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -16,40 +15,7 @@ pub(crate) async fn publish_definitions( snapshot: &SnapshotRecord, definitions: Definitions, ) -> Result { - for table in ["sqb_job_nodes", "sqb_jobs"] { - (transaction, _) = query( - transaction, - &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), - vec![ - Parameter::Id(snapshot.location_id), - Parameter::Text(snapshot.target.clone()), - ], - ) - .await?; - } - let jobs = definitions - .jobs - .into_iter() - .map(|(name, job)| (name, "job", job)) - .chain( - definitions - .groups - .into_iter() - .map(|(name, group)| (name, "group", group.job)), - ); - for (name, kind, job) in jobs { - let definition = - serde_json::to_value(&job).map_err(|error| ControlError::Invalid(error.to_string()))?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", - vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), - Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; - for node in job.selection { - (transaction, _) = query(transaction, - "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", - vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node)]).await?; - } - } + transaction = publish_jobs(transaction, snapshot, &definitions).await?; (transaction, _) = query(transaction, "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; @@ -68,29 +34,7 @@ pub(crate) async fn publish_definitions( .next() .map(decode_row::) .transpose()?; - if let Some(state) = &state - && let Schedule::Calendar { - times: previous_times, - .. - } = &state.definition_json.schedule - && let Schedule::Calendar { times, .. } = &mut trigger.schedule - { - times.extend( - previous_times - .iter() - .filter(|time| { - **time >= state.started_at - && **time <= snapshot.published_at - && state.last_tick_for.is_none_or(|last| **time > last) - }) - .copied(), - ); - if times.len() > MAX_TICKS_PER_BATCH { - return Err(ControlError::Invalid(format!( - "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" - ))); - } - } + retain_calendar_ticks(&name, &mut trigger, state.as_ref(), snapshot)?; let definition = serde_json::to_value(&trigger) .map_err(|error| ControlError::Invalid(error.to_string()))?; (transaction, _) = query(transaction, @@ -115,3 +59,45 @@ pub(crate) async fn publish_definitions( } Ok(transaction) } + +async fn publish_jobs( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + } + let jobs = definitions + .jobs + .iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .iter() + .map(|(name, group)| (name, "group", &group.job)), + ); + for (name, kind, job) in jobs { + let definition = + serde_json::to_value(job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; + for node in &job.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node.clone())]).await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 2c05267eb7..4c388072ab 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -14,7 +14,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::helpers::{snapshot, trigger_audits}; use crate::scheduler::tests::contracts::ticks::test_types::{ - TransactionFault, TriggerControlsTestCase, + TickTransactionTestCase, TransactionFault, TriggerControlsTestCase, }; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; @@ -253,10 +253,9 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur { use crate::runs::main::list_runs::list_runs; use crate::scheduler::main::list_ticks::list_ticks; - use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; use serde_json::json; - for test_case in [ + let test_cases = [ TickTransactionTestCase { description: "SQLite retains due calendar ticks", backend: Backend::Sqlite, @@ -269,7 +268,8 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur expected_ticks: 3, expected_runs: 1, }, - ] { + ]; + for test_case in test_cases { run(async { let Some(fixture) = open_test_store(test_case.backend).await else { return; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index bbb369fa17..8edfb3a580 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -131,8 +131,7 @@ pub(crate) fn build_orchestration_snapshot( input = compile_declarations( py, input, - &project_dir, - graph.bind(py).get().graph(), + (&project_dir, graph.bind(py).get().graph()), authored, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 7bc75bb723..9706fcc399 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; @@ -60,8 +60,7 @@ pub(crate) fn discover_declarations<'py>( pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - project_dir: &Path, - graph: &ProjectGraph, + (project_dir, graph): (&Path, &ProjectGraph), declarations: Vec, ) -> PyResult { if declarations.is_empty() { @@ -78,7 +77,7 @@ pub(crate) fn compile_declarations( let mut jobs: BTreeMap = BTreeMap::new(); let mut authored_groups: Vec = Vec::new(); let mut authored_jobs: Vec = Vec::new(); - let mut authored_calendars = Vec::new(); + let mut authored_calendars: Vec> = Vec::new(); for declaration in declarations { match declaration { AuthoredDeclaration::Job(job) => authored_jobs.push(job), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 6f9e92f4f8..d2efd5db0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compiled_project::snapshot::{ NativeOrchestrationSnapshot, build_orchestration_snapshot, }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; +use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::worker::execute_worker; use crate::bindings::_helpers::native_host::host::write_console; @@ -50,7 +50,7 @@ fn metadata( let generation = arguments .value("generation")? .extract::>()? - .or_else(|| (action.as_deref() == Some("execute")).then_some(1)) + .or_else(|| (action.as_deref() == Some(EXECUTE_ACTION)).then_some(1)) .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; if target.trim().is_empty() || generation < 1 { return Err(PyValueError::new_err( @@ -132,11 +132,10 @@ fn compile_location<'py>( view, &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, project_dir.extract()?, - // Workers verify nodes and environment, not dynamic calendar declarations. arguments .value("action")? .extract::>()? - .is_some_and(|action| action == "execute") + .is_some_and(|action| action == EXECUTE_ACTION) .then(Vec::new), )?; status(py, "Location snapshot compiled.")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index dc0a1d6e2d..ffee78c649 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -1,2 +1,5 @@ /// Location actions accepted by the interpreter-side one-shot publisher. pub(crate) const PUBLISH_ACTION: &str = "publish"; + +/// Workers verify the pinned nodes and environment without reevaluating declarations. +pub(crate) const EXECUTE_ACTION: &str = "execute"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs new file mode 100644 index 0000000000..5c0e4cbe43 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -0,0 +1,4 @@ +//! Authoring parameters reserved by the code-location host. + +/// The calendar's evaluation window is supplied by the host, not a provider. +pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index e604e71354..ad5619410e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,6 +1,7 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. pub(crate) mod connections; +pub(crate) mod constants; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 198541d498..c5366827ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -18,6 +18,7 @@ use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::CALENDAR_WINDOW_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::authored_setting; @@ -94,9 +95,10 @@ fn horizon(value: &str) -> PyResult { ] .into_iter() .try_fold(0_i64, |total, (digits, factor)| { - total.checked_add(digits.parse::().ok()?.checked_mul(factor)?) - }) - .ok_or_else(invalid)?; + let digits = digits.parse::().map_err(|_| invalid())?; + let seconds = digits.checked_mul(factor).ok_or_else(invalid)?; + total.checked_add(seconds).ok_or_else(invalid) + })?; TimeDelta::try_seconds(seconds).ok_or_else(invalid) } @@ -132,10 +134,10 @@ fn calendar_times( .call((), Some(&window))?; let function = authored.getattr("function")?; let arguments = PyDict::new(py); - arguments.set_item("window", window)?; + arguments.set_item(CALENDAR_WINDOW_PARAMETER, window)?; for usage in provider_usages(&function, providers)?.try_iter()? { let usage = usage?; - if usage.getattr("parameter_name")?.extract::()? == "window" { + if usage.getattr("parameter_name")?.extract::()? == CALENDAR_WINDOW_PARAMETER { continue; } arguments.set_item( @@ -143,7 +145,7 @@ fn calendar_times( session.call_method1("get", (usage.getattr("provider_name")?,))?, )?; } - let mut times = BTreeSet::new(); + let mut times: BTreeSet> = BTreeSet::new(); for (index, time) in function .call((), Some(&arguments))? .try_iter()? @@ -186,7 +188,7 @@ pub(crate) fn compile_calendars( authored: Vec>, mut definitions: Definitions, ) -> PyResult { - let mut calendars = Vec::new(); + let mut calendars: Vec> = Vec::new(); for calendar in authored { let targets = calendar .bind(py) From 5fa350dcabf3811c897b7bc3f6621084f5d286eb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:23:59 +0000 Subject: [PATCH 046/138] feat: deliver platform health to server webhook --- .../src/native_commands/_helpers/execution.rs | 47 +++++ .../src/native_commands/main/_run_control.rs | 17 +- .../src/workers/_helpers/invocation.rs | 2 + .../src/platform/_helpers/mod.rs | 1 + .../src/platform/_helpers/webhook_delivery.rs | 61 ++++++ .../src/platform/constants.rs | 8 + .../platform/main/connect_platform_webhook.rs | 31 +++ .../src/platform/main/mod.rs | 2 + .../src/platform/main/run_platform_sink.rs | 48 +++++ .../src/platform/mod.rs | 1 + .../src/platform/models.rs | 30 +++ .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/platform_sink.sql | 7 + .../tests/api/test_location_work.rs | 4 +- .../tests/platform/helpers.rs | 104 +++++++++- .../tests/platform/test_contracts.rs | 2 + .../tests/platform/test_types.rs | 23 ++- .../tests/platform/test_webhook.rs | 193 ++++++++++++++++++ .../tests/snapshots/helpers.rs | 3 +- fensu.toml | 6 + .../publication/docker/helpers.py | 2 + .../compiled_project/publication/helpers.py | 9 +- .../publication/test_docker_worker.py | 2 +- 24 files changed, 583 insertions(+), 24 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs create mode 100644 crates/sqlbuild-orchestration/src/store/platform_sink.sql create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_webhook.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index dac117b46c..27c20ee723 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -8,7 +8,11 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink; +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; @@ -300,6 +304,49 @@ fn location_child(options: &LocationServe, script: &Path) -> Command { command } +/// Health production and delivery remain native and independent of scheduler leadership. +pub(crate) fn start_platform_roles( + tasks: &mut JoinSet>, + (store, shutdown): (&ControlStore, &watch::Receiver), +) -> Result<(), ControlError> { + let sink = match std::env::var(PLATFORM_WEBHOOK_VARIABLE) { + Ok(endpoint) => Some(connect_platform_webhook(&endpoint)?), + Err(std::env::VarError::NotPresent) => None, + Err(_) => { + return Err(ControlError::Invalid( + "platform webhook configuration must be UTF-8".into(), + )); + } + }; + drop(write_output(&[( + Stream::Stderr, + "Starting platform watchdog...\n", + )])); + let watchdog_store = store.clone(); + let watchdog_shutdown = shutdown.clone(); + tasks.spawn( + async move { run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await }, + ); + drop(write_output(&[( + Stream::Stderr, + "Platform watchdog started\n", + )])); + if let Some(sink) = sink { + drop(write_output(&[( + Stream::Stderr, + "Starting platform webhook...\n", + )])); + let sink_store = store.clone(); + let sink_shutdown = shutdown.clone(); + tasks.spawn(async move { run_platform_sink(&sink_store, sink, sink_shutdown).await }); + drop(write_output(&[( + Stream::Stderr, + "Platform webhook started\n", + )])); + } + Ok(()) +} + /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. pub(crate) async fn open_control_store( namespace: &Namespace, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 1ca4f4f3b1..02e6928bd2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,7 +1,6 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -11,7 +10,7 @@ use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{control_address, flag}; use crate::native_commands::_helpers::execution::{ - open_control_store, open_control_workers, supervise_control_roles, + open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; use crate::native_commands::_helpers::terminal::write_output; @@ -48,19 +47,7 @@ pub(crate) async fn run_control( listener, stopped(shutdown.clone()), )); - drop(write_output(&[( - Stream::Stderr, - "Starting platform watchdog...\n", - )])); - let watchdog_store = store.clone(); - let watchdog_shutdown = shutdown.clone(); - tasks.spawn(async move { - run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await - }); - drop(write_output(&[( - Stream::Stderr, - "Platform watchdog started\n", - )])); + start_platform_roles(&mut tasks, (&store, &shutdown))?; } if standalone || !flag(namespace, "control_no_scheduler") { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs index 67f3054326..a951cc74d6 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -1,6 +1,7 @@ //! Both launchers hand off the same scoped invocation and exclude privileged credentials. use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::runs::models::RunClaim; use crate::workers::models::LauncherScope; @@ -26,6 +27,7 @@ pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { name == control_variable + || name == PLATFORM_WEBHOOK_VARIABLE || matches!( name, "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index 8f5f6285fa..06cdb8c059 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod location_heartbeat; pub(crate) mod location_work; pub(crate) mod observations; pub(crate) mod watchdog; +pub(crate) mod webhook_delivery; pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs new file mode 100644 index 0000000000..73879b7c63 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs @@ -0,0 +1,61 @@ +//! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. + +use crate::platform::constants::SINK_PAGE_SIZE; +use crate::platform::models::{PlatformEventEnvelope, PlatformWebhook}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A lost acknowledgement may resend the same event_id; receivers must deduplicate by that ID. +pub(crate) async fn deliver_platform_events( + store: &ControlStore, + sink: &PlatformWebhook, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT storage_id, event_id, occurred_at, payload FROM recall_events \ + WHERE producer='platform' AND invocation_id LIKE 'platform/%' AND storage_id > \ + (SELECT COALESCE(MAX(storage_id), 0) FROM sqb_platform_sink_deliveries WHERE sink_id=$1) \ + ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Text(sink.sink_id.clone()), + Parameter::Integer(SINK_PAGE_SIZE), + ], + ) + .await?; + let events: Vec = + rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + let count = events.len(); + for event in events { + let response = sink + .client + .post(sink.url.clone()) + .header("Idempotency-Key", event.event_id.to_string()) + .json(&event) + .send() + .await + .map_err(|_| { + ControlError::Invalid("platform webhook delivery failed; event retained".into()) + })?; + if !response.status().is_success() { + return Err(ControlError::Invalid(format!( + "platform webhook returned HTTP {}; event retained", + response.status().as_u16() + ))); + } + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_platform_sink_deliveries (sink_id, storage_id, event_id, delivered_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (sink_id, storage_id) DO NOTHING", + vec![Parameter::Text(sink.sink_id.clone()), Parameter::Integer(event.storage_id), + Parameter::Text(event.event_id.to_string()), Parameter::Time(now)]).await?; + commit(transaction).await?; + } + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs new file mode 100644 index 0000000000..f405071bd8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -0,0 +1,8 @@ +//! Bounds shared by the platform event reader and webhook dispatcher. + +/// Operator-owned delivery endpoint; never inherited by project workers or printed in logs. +pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; + +pub(crate) const SINK_PAGE_SIZE: i64 = 100; +pub(crate) const SINK_REQUEST_TIMEOUT_SECONDS: u64 = 10; +pub(crate) const SINK_POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs new file mode 100644 index 0000000000..4a715b9ec8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs @@ -0,0 +1,31 @@ +//! Validate operator-owned delivery configuration without exposing credentials in errors. + +use std::time::Duration; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::platform::constants::SINK_REQUEST_TIMEOUT_SECONDS; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; + +/// Changing the endpoint creates a new consumer, starting from the retained event history. +pub fn connect_platform_webhook(endpoint: &str) -> Result { + let url = reqwest::Url::parse(endpoint) + .map_err(|_| ControlError::Invalid("platform webhook needs an HTTP(S) URL".into()))?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err(ControlError::Invalid( + "platform webhook needs an HTTP(S) URL".into(), + )); + } + let sink_id = sha256_hex(url.as_str().as_bytes()); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(SINK_REQUEST_TIMEOUT_SECONDS)) + .build() + .map_err(|_| ControlError::Invalid("platform webhook client could not start".into()))?; + Ok(PlatformWebhook { + client, + url, + sink_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 095c42262e..e9b14619b3 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,9 +1,11 @@ pub mod claim_location_work; pub mod complete_location_work; pub mod connect_location_work; +pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; +pub mod run_platform_sink; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs b/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs new file mode 100644 index 0000000000..02757c1627 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs @@ -0,0 +1,48 @@ +//! Deliver platform health independently of user interpreters and scheduler leadership. + +use std::time::Duration; + +use tokio::sync::watch; + +use crate::platform::_helpers::webhook_delivery::deliver_platform_events; +use crate::platform::constants::SINK_POLL_SECONDS; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_platform_sink( + store: &ControlStore, + sink: PlatformWebhook, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + let mut failed = false; + while !*shutdown.borrow() { + tokio::select! { + result = deliver_platform_events(store, &sink) => { + match result { + Ok(count) => { + if failed { + eprintln!("Platform webhook delivery recovered"); + } + failed = false; + if count > 0 { continue; } + }, + Err(_) => { + if !failed { + eprintln!("Platform webhook delivery failed; retaining events and retrying"); + } + failed = true; + }, + } + }, + result = shutdown.changed() => { + if result.is_err() || *shutdown.borrow() { break; } + }, + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(SINK_POLL_SECONDS)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs index 32d92d3cc2..d58521e0f6 100644 --- a/crates/sqlbuild-orchestration/src/platform/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 29f902dff0..72340bfd81 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -152,3 +152,33 @@ pub(crate) struct WatchdogTrigger { pub(crate) definition_json: TriggerDefinition, pub(crate) next_due_at: Option>, } + +/// The endpoint may contain credentials; neither errors nor Debug reveal it. +#[derive(Clone)] +pub struct PlatformWebhook { + pub(crate) client: reqwest::Client, + pub(crate) url: reqwest::Url, + pub(crate) sink_id: String, +} + +impl std::fmt::Debug for PlatformWebhook { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PlatformWebhook") + .finish_non_exhaustive() + } +} + +/// Stable event identity permits receiver-side deduplication after uncertain acknowledgements. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventEnvelope { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub payload: PlatformEventPayload, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventPayload { + pub platform: PlatformEvent, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index cd44c4e59a..fd922e7d2f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -9,6 +9,7 @@ use crate::store::types::Row; pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); +pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 31fefb1c4e..56f2194ffb 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, SCHEMA, render, verify_migrations, + LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -30,6 +30,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { let additions = [ ("location_instances", LOCATION_INSTANCES), ("location_work", LOCATION_WORK), + ("platform_sink", PLATFORM_SINK), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/platform_sink.sql b/crates/sqlbuild-orchestration/src/store/platform_sink.sql new file mode 100644 index 0000000000..ece8bb3b6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/platform_sink.sql @@ -0,0 +1,7 @@ +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at {time} NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 03d21e27b5..7d77cdf9fa 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version=3").await; + execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index a00d9adf24..e2da7ca896 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -1,17 +1,29 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; -use crate::platform::models::PlatformEvent; +use axum::Json; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::models::{RecallBatch, RecallRecord}; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; use crate::store::tests::helpers::read_recall_events; use chrono::{TimeDelta, Utc}; use serde_json::json; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::schema::models::Table; use tokio::task::JoinHandle; use uuid::Uuid; -use crate::platform::tests::test_types::Observation; +use crate::platform::tests::test_types::{Observation, WebhookReceiver}; +use crate::runs::tests::helpers::{queue, request}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { let future = Utc::now() + TimeDelta::days(10); @@ -109,3 +121,91 @@ pub(crate) async fn wait_for_scans( .await .expect("both replicas finish a paged scan without a scheduler") } + +pub(crate) async fn receive_webhook( + State(receiver): State, + headers: HeaderMap, + Json(event): Json, +) -> impl IntoResponse { + assert_eq!( + headers.get("Idempotency-Key").expect("deduplication key"), + event.event_id.to_string().as_str() + ); + let mut state = receiver.state.lock().expect("receiver state"); + state.requests.push(event); + receiver.delivered.send_replace(state.requests.len()); + ( + StatusCode::from_u16(state.status).expect("configured response status"), + [(axum::http::header::LOCATION, "/events")], + ) +} + +pub(crate) async fn wait_for_webhook( + receiver: &WebhookReceiver, + dispatcher: &JoinHandle>, + count: usize, +) { + let mut delivered = receiver.delivered.subscribe(); + tokio::time::timeout( + Duration::from_secs(10), + delivered.wait_for(|received| *received >= count), + ) + .await + .expect("dispatcher drains more than one bounded page") + .expect("webhook receiver remains available"); + assert!( + !dispatcher.is_finished(), + "delivery loop stopped before draining" + ); +} + +pub(crate) async fn publish_platform_impostor(store: &ControlStore, identity: (Uuid, Uuid)) { + let mut requested = request(identity, 42, 0, &["orders"]); + requested.image_id = None; + queue(store, &requested).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let batch = RecallBatch { + invocation_id: "platform/impostor".into(), + records: vec![RecallRecord { + table: Table::Events, + values: BTreeMap::from([ + ( + "event_id".into(), + RecallValue::Text(Uuid::now_v7().to_string()), + ), + ( + "event_type".into(), + RecallValue::Text("platform_heartbeat".into()), + ), + ("event_schema_version".into(), RecallValue::Integer(1)), + ("event_kind".into(), RecallValue::Text("runtime".into())), + ("severity".into(), RecallValue::Text("info".into())), + ( + "occurred_at".into(), + RecallValue::Text("2026-01-01T00:00:00Z".into()), + ), + ("producer".into(), RecallValue::Text("platform".into())), + ("invocation_sequence".into(), RecallValue::Integer(0)), + ( + "payload".into(), + RecallValue::Text( + json!({"platform": {"kind": "heartbeat", "replica_id": Uuid::now_v7()}}) + .to_string(), + ), + ), + ]), + }], + }; + let result = publish_run_recall(store, (claim.request.run_id, claim.claim_id), &batch) + .await + .expect("publish untrusted producer"); + assert_eq!(result.delivered, [(Table::Events, 1)]); + assert!(result.conflicts.is_empty()); +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs index 1a231c8e58..0f5c53a679 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -6,3 +6,5 @@ mod helpers; mod test_types; #[path = "test_watchdog.rs"] mod test_watchdog; +#[path = "test_webhook.rs"] +mod test_webhook; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 01cf50f419..970ec7a8c0 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -1,4 +1,6 @@ -use crate::platform::models::PlatformEvent; +use std::sync::{Arc, Mutex}; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; use chrono::{DateTime, Utc}; use serde::Deserialize; @@ -24,3 +26,22 @@ pub(crate) struct Observation { pub(crate) struct PlatformPayload { pub(crate) platform: PlatformEvent, } + +pub(crate) struct PlatformSinkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) initial_status: u16, + pub(crate) expected_events: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct WebhookReceiver { + pub(crate) state: Arc>, + pub(crate) delivered: tokio::sync::watch::Sender, +} + +#[derive(Debug)] +pub(crate) struct WebhookState { + pub(crate) status: u16, + pub(crate) requests: Vec, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs new file mode 100644 index 0000000000..d2aeeea1c8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs @@ -0,0 +1,193 @@ +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::routing::post; +use uuid::Uuid; + +use crate::platform::_helpers::webhook_delivery::deliver_platform_events; +use crate::platform::main::connect_platform_webhook::connect_platform_webhook; +use crate::platform::main::run_platform_sink::run_platform_sink; +use crate::platform::main::run_watchdog::run_watchdog; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; + +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{ + publish_platform_impostor, receive_webhook, snapshot, wait_for_scans, wait_for_webhook, +}; +use crate::platform::tests::test_types::{PlatformSinkTestCase, WebhookReceiver, WebhookState}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_survive_restart_and_lost_acknowledgements() + { + let test_cases = [ + PlatformSinkTestCase { + description: "SQLite durable platform delivery", + backend: Backend::Sqlite, + initial_status: 503, + expected_events: 105, + }, + PlatformSinkTestCase { + description: "PostgreSQL durable platform delivery", + backend: Backend::Postgres, + initial_status: 307, + expected_events: 105, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate delivery store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish schedules"); + // No location process and no scheduler are running. + let replica_id = Uuid::now_v7(); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let watchdog_store = store.clone(); + let tasks = [tokio::spawn(async move { + run_watchdog(&watchdog_store, replica_id, shutdown).await + })]; + let observations = wait_for_scans(store, &BTreeSet::from([replica_id]), &tasks).await; + stop.send(true).expect("stop watchdog"); + for task in tasks { + task.await + .expect("watchdog task") + .expect("watchdog stopped"); + } + assert_eq!( + observations.len(), + test_case.expected_events, + "{}", + test_case.description + ); + publish_platform_impostor(store, (location, published.snapshot_id)).await; + + let state = Arc::new(Mutex::new(WebhookState { + status: test_case.initial_status, + requests: Vec::new(), + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("webhook listener"); + let endpoint = format!( + "http://delivery:sample-password@{}/events?token=sample-token", + listener.local_addr().expect("address") + ); + let receiver = WebhookReceiver { + state: state.clone(), + delivered: tokio::sync::watch::channel(0).0, + }; + let app = Router::new() + .route("/events", post(receive_webhook)) + .with_state(receiver.clone()); + let (http_stop, http_shutdown) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + drop(http_shutdown.await); + }) + .await + }); + let sink = connect_platform_webhook(&endpoint).expect("configure webhook"); + let diagnostic = format!("{sink:?}"); + assert!(!diagnostic.contains("sample-password")); + assert!(!diagnostic.contains("sample-token")); + assert!(deliver_platform_events(store, &sink).await.is_err()); + assert_eq!( + state.lock().expect("receiver").requests.len(), + 1, + "failure stops the page" + ); + state.lock().expect("receiver").status = 204; + let fault = TransactionFault { + description: "lose a successful webhook acknowledgement", + table: "sqb_platform_sink_deliveries", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'test receipt failure')", + postgres_body: "RAISE EXCEPTION 'test receipt failure'", + }; + install_fault(store, &fault).await; + assert!(deliver_platform_events(store, &sink).await.is_err()); + remove_fault(store, &fault).await; + let second = second_store(store).await; + let restarted = connect_platform_webhook(&endpoint).expect("restart webhook client"); + let (sink_stop, sink_shutdown) = tokio::sync::watch::channel(false); + let sink_store = second.clone(); + let dispatcher = tokio::spawn(async move { + run_platform_sink(&sink_store, restarted, sink_shutdown).await + }); + wait_for_webhook(&receiver, &dispatcher, test_case.expected_events + 2).await; + sink_stop.send(true).expect("stop dispatcher"); + tokio::time::timeout(Duration::from_secs(2), dispatcher) + .await + .expect("bounded dispatcher shutdown") + .expect("dispatcher task") + .expect("dispatcher stopped"); + // Shutdown may cancel the last receipt after the remote endpoint accepted it. + assert!( + deliver_platform_events(store, &sink) + .await + .expect("reconcile in-flight delivery") + <= 1 + ); + assert_eq!( + deliver_platform_events(store, &sink) + .await + .expect("durable cursor"), + 0 + ); + { + let receiver = state.lock().expect("receiver"); + let requests = &receiver.requests; + assert!( + (test_case.expected_events + 2..=test_case.expected_events + 3) + .contains(&requests.len()) + ); + assert_eq!(requests[0], requests[1], "HTTP retry retains full identity"); + assert_eq!( + requests[1], requests[2], + "lost receipt retries the same fact" + ); + assert_eq!( + requests[2..] + .iter() + .map(|event| event.event_id) + .collect::>() + .len(), + test_case.expected_events + ); + let mut delivered = requests[2..].to_vec(); + delivered.dedup(); + assert_eq!(delivered.len(), observations.len()); + for (event, observation) in delivered.iter().zip(&observations) { + assert_eq!(event.storage_id, observation.storage_id); + assert_eq!(event.occurred_at, observation.occurred_at); + assert_eq!(event.payload.platform, observation.payload.platform); + } + } + http_stop.send(()).expect("stop receiver"); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .expect("bounded shutdown") + .expect("server task") + .expect("server stopped"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 7c093b292f..7b2283fbd1 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ + "DROP TABLE sqb_platform_sink_deliveries; \ + DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", ) .expect("reconstruct the previous schema without changing existing facts"); diff --git a/fensu.toml b/fensu.toml index aefc72b446..8c5126e9c4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1570,6 +1570,12 @@ path = "crates/sqlbuild-orchestration/src/credentials/models.rs" symbols = ["IssuedToken"] reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/platform/models.rs" +symbols = ["PlatformWebhook"] +reason = "PlatformWebhook implements Debug manually to redact its endpoint, which may contain delivery credentials; other models derive Debug." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py index 58135a9dec..d1ac635fdd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -89,6 +89,8 @@ def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeploy "--env", "SQLBUILD_CONTROL_DATABASE_URL=synthetic-control-store", "--env", + "SQLBUILD_PLATFORM_WEBHOOK_URL=http://127.0.0.1:1/events?token=synthetic-delivery-credential", + "--env", "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", *mounts, image_id, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index f0717f1000..bcc8855eca 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -169,6 +169,7 @@ def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: log: Path = directory / "server.log" environment: dict[str, str] = os.environ.copy() environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) with log.open("wb") as stderr: process: subprocess.Popen[bytes] = subprocess.Popen( [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", "--no-scheduler"], @@ -271,7 +272,9 @@ def scheduled_worker( "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" ), "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' @@ -334,6 +337,10 @@ def scheduled_worker( with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") + monkeypatch.setenv( + "SQLBUILD_PLATFORM_WEBHOOK_URL", + "http://127.0.0.1:1/events?token=synthetic-delivery-credential", + ) log: Path = directory / "scheduler.log" with location_process( [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index a09544ebf1..8d54cf22ee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -47,7 +47,7 @@ "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " - "'SQLBUILD_SCHEDULER_TEST_STORE'):\n" + "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL'):\n" " assert name not in os.environ, name\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' From c302e338e56ed2e1cf43f042f63825e36c805c30 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:31:17 +0000 Subject: [PATCH 047/138] test: share disposable PostgreSQL harness across native gates --- CONTRIBUTING.md | 13 ++++++++ Makefile | 8 ++++- .../tests/store/helpers.rs | 11 ++----- scripts/native_tests/__init__.py | 1 + scripts/native_tests/constants.py | 6 ++++ scripts/native_tests/main/__init__.py | 1 + scripts/native_tests/main/run.py | 33 +++++++++++++++++++ scripts/run_native_tests.py | 12 +++++++ 8 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 scripts/native_tests/__init__.py create mode 100644 scripts/native_tests/constants.py create mode 100644 scripts/native_tests/main/__init__.py create mode 100644 scripts/native_tests/main/run.py create mode 100644 scripts/run_native_tests.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f0ab39173e..ba45c64ca6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,19 @@ For a change confined to one Rust crate, run its focused tests before rebuilding cargo test -p sqlbuild-sqltext given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern ``` +Control-plane integration tests require Docker and run both SQLite and PostgreSQL: + +```bash +make test-native NATIVE_TEST_ARGS='-p sqlbuild-orchestration --lib' +``` + +The same Testcontainers harness runs under `make rust-check` in CI. Each invocation starts one +digest-pinned PostgreSQL container, waits for database readiness, shares it across the suite with +an isolated schema per fixture, and removes it after success or failure. No host PostgreSQL setup +is needed. Missing Docker, startup failures or missing database configuration fail the gate; +PostgreSQL cases never silently skip. `NATIVE_TEST_ARGS` accepts Cargo test arguments; its default +runs the workspace with all features. + Keep the toolchain, `CARGO_TARGET_DIR`, profile and `RUSTFLAGS` consistent between iterations. Changing them can invalidate compiled artifacts. Coordinate builds that share a target directory; do not delete another task's artifacts or change global compiler-cache settings. The first build diff --git a/Makefile b/Makefile index 1753e39f05..911e79832f 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,13 @@ fast-native: rust-check: rust-lint uv run fensu check - cargo test --workspace --all-features + $(MAKE) test-native + +NATIVE_TEST_ARGS ?= --workspace --all-features + +.PHONY: test-native +test-native: + uv run --no-sync python -m scripts.run_native_tests $(NATIVE_TEST_ARGS) .PHONY: rust-lint rust-lint: diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index db2525d17d..ba5fd692cc 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -32,14 +32,9 @@ pub(crate) async fn open_test_store(backend: Backend) -> Option { postgres: None, }); }; - let Ok(url) = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL") else { - assert!( - std::env::var_os("SQLBUILD_REQUIRE_POSTGRES_TESTS").is_none(), - "PostgreSQL test URL is required" - ); - eprintln!("PostgreSQL integration requires SQLBUILD_TEST_CONTROL_POSTGRES_URL"); - return None; - }; + let url = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL").expect( + "PostgreSQL integration requires the disposable database harness; run make test-native", + ); let admin = sqlx_postgres::PgPool::connect(&url) .await .expect("connect to test PostgreSQL"); diff --git a/scripts/native_tests/__init__.py b/scripts/native_tests/__init__.py new file mode 100644 index 0000000000..4175a3327b --- /dev/null +++ b/scripts/native_tests/__init__.py @@ -0,0 +1 @@ +"""Shared native integration-test infrastructure.""" diff --git a/scripts/native_tests/constants.py b/scripts/native_tests/constants.py new file mode 100644 index 0000000000..3473a3cec7 --- /dev/null +++ b/scripts/native_tests/constants.py @@ -0,0 +1,6 @@ +"""Reproducible native integration database image.""" + +POSTGRES_IMAGE: str = ( + "postgres:17.6-alpine3.22@" + "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94" +) diff --git a/scripts/native_tests/main/__init__.py b/scripts/native_tests/main/__init__.py new file mode 100644 index 0000000000..22055e8817 --- /dev/null +++ b/scripts/native_tests/main/__init__.py @@ -0,0 +1 @@ +"""Native test command entrypoints.""" diff --git a/scripts/native_tests/main/run.py b/scripts/native_tests/main/run.py new file mode 100644 index 0000000000..fa3a52fc5e --- /dev/null +++ b/scripts/native_tests/main/run.py @@ -0,0 +1,33 @@ +"""One disposable database per native suite; Rust fixtures isolate their own schemas.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE + + +def run_native_tests(argv: list[str] | None = None) -> int: + """Fail closed on Docker/startup errors and propagate Cargo's result after cleanup.""" + arguments: list[str] = sys.argv[1:] if argv is None else argv + print("Starting disposable PostgreSQL for native integration tests...", flush=True) + try: + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + print("PostgreSQL ready. Running native tests with isolated schemas...", flush=True) + result: subprocess.CompletedProcess[bytes] = subprocess.run( + ["cargo", "test", *arguments], + env=os.environ + | { + "SQLBUILD_TEST_CONTROL_POSTGRES_URL": postgres.get_connection_url(), + }, + check=False, + ) + print(f"PostgreSQL removed. Native tests exited {result.returncode}.", flush=True) + return result.returncode + except Exception as error: + print(f"Native integration tests failed: {error}", file=sys.stderr, flush=True) + return 1 diff --git a/scripts/run_native_tests.py b/scripts/run_native_tests.py new file mode 100644 index 0000000000..2f673d1fc7 --- /dev/null +++ b/scripts/run_native_tests.py @@ -0,0 +1,12 @@ +"""Direct wrapper for native tests with disposable PostgreSQL.""" + +from scripts.native_tests.main.run import run_native_tests + + +def main(argv: list[str] | None = None) -> int: + """Run Cargo tests with a shared Testcontainers PostgreSQL instance.""" + return run_native_tests(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) From 2c00edb44c9aa511c27a714650f60fad545b2e51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:53:53 +0000 Subject: [PATCH 048/138] feat: schedule durable built-in platform sensors Queue sensor runs, commit cursors and publish outcomes atomically under scheduler fencing. Preserve deduplication, recovery and operator overrides across deployments. Place the public PlatformEventKind enum in types.py with an enum-only rule-3 facade entry, retaining remaining_lines=0 and native implementation ownership. Cover mixed schedule/sensor publication and real worker execution without relaxing existing regression assertions. --- .../orchestration/platform_orders.py | 30 +++ .../orchestration/edge_cases/definitions.py | 18 +- .../src/declarations/_helpers/documents.rs | 22 +- .../src/declarations/_helpers/mod.rs | 1 + .../declarations/_helpers/operator_state.rs | 81 +++++++ .../src/declarations/main/change_trigger.rs | 44 +--- .../declarations/main/publish_definitions.rs | 2 + .../declarations/main/snapshot_definitions.rs | 2 + .../src/declarations/models.rs | 14 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/watchdog.rs | 2 +- .../src/platform/_helpers/webhook_delivery.rs | 20 +- .../src/platform/constants.rs | 2 +- .../src/platform/main/mod.rs | 1 + .../src/platform/main/read_platform_events.rs | 22 ++ .../src/platform/models.rs | 21 ++ .../src/runs/_helpers/requests/job.rs | 68 +----- .../src/runs/main/frozen_job_request.rs | 70 ++++++ .../src/runs/main/mod.rs | 1 + .../src/runs/main/request_job_run.rs | 2 +- .../scheduler/_helpers/drain_due_triggers.rs | 5 +- .../src/scheduler/_helpers/drain_sensors.rs | 36 +++ .../src/scheduler/_helpers/mod.rs | 1 + .../src/sensors/_helpers/attempts.rs | 65 ++++++ .../src/sensors/_helpers/facts.rs | 65 ++++++ .../src/sensors/_helpers/mod.rs | 4 + .../src/sensors/_helpers/platform.rs | 34 +++ .../src/sensors/_helpers/requests.rs | 62 +++++ .../sensors/main/begin_sensor_evaluation.rs | 80 +++++++ .../sensors/main/complete_builtin_sensor.rs | 73 ++++++ .../src/sensors/main/mod.rs | 6 + .../src/sensors/main/publish_sensors.rs | 37 +++ .../src/sensors/main/read_sensor_cursor.rs | 24 ++ .../src/sensors/main/reset_sensor_cursor.rs | 41 ++++ .../src/sensors/main/validate_sensors.rs | 22 ++ .../sqlbuild-orchestration/src/sensors/mod.rs | 9 + .../src/sensors/models.rs | 114 +++++++++ .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/sensors.sql | 25 ++ .../tests/api/test_location_work.rs | 4 +- .../tests/scheduler/ticks/test_triggers.rs | 4 +- .../tests/sensors/helpers.rs | 114 +++++++++ .../tests/sensors/test_contracts.rs | 10 + .../tests/sensors/test_delivery.rs | 183 +++++++++++++++ .../tests/sensors/test_recovery.rs | 218 ++++++++++++++++++ .../tests/sensors/test_types.rs | 7 + .../tests/snapshots/helpers.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 41 +++- .../_helpers/location_command/command.rs | 15 +- .../native_host/orchestration_declarations.rs | 35 ++- .../native_host/orchestration_triggers.rs | 36 ++- python-boundary.toml | 5 + src/sqlbuild/_native.pyi | 2 +- src/sqlbuild/orchestration.py | 10 + .../main/compile_snapshot.py | 4 +- .../orchestration_declarations/models.py | 23 ++ .../orchestration_declarations/types.py | 12 + .../compiler/compiled_project/_test_types.py | 2 + .../publication/_test_types.py | 1 + .../compiled_project/publication/helpers.py | 3 + .../publication/test_calendar.py | 10 +- .../publication/test_publication.py | 19 +- .../publication/test_worker.py | 41 ++++ .../test_orchestration_snapshot.py | 82 +++++++ 65 files changed, 1854 insertions(+), 161 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/platform_orders.py create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/models.rs create mode 100644 crates/sqlbuild-orchestration/src/store/sensors.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_types.rs create mode 100644 src/sqlbuild/orchestration_declarations/types.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py new file mode 100644 index 0000000000..4814f4ae1e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py @@ -0,0 +1,30 @@ +"""Sensor-only roots retain referenced jobs while target filters and event enums stay native.""" + +from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor + + +def definitions(): + refresh = job("health_orders", description="Refresh orders.", select="orders") + return [ + sensor( + "overdue_orders", + description="Catch up overdue order reporting.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run=refresh, + minimum_interval_seconds=7, + default_status="stopped", + targets=("development",), + ), + sensor( + "other_target_orders", + description="Refresh another target.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run=refresh, + targets=("production",), + ), + ] + + +triggers = definitions() +alias = triggers[0] +triggers.append(triggers) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index af566e9dff..ef8caed097 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -1,6 +1,13 @@ """Exercise alias deduplication and cyclic factory containers during native discovery.""" -from sqlbuild.orchestration import job, job_group, schedule +from sqlbuild.orchestration import ( + PlatformEventKind, + job, + job_group, + platform_event, + schedule, + sensor, +) edge_orders = job( "edge_snapshot_orders", @@ -51,3 +58,12 @@ jobs=[edge_products], concurrency_groups=("edge_products_limit",), ) +edge_health = sensor( + "edge_platform_orders", + description="Catch up retained platform observations with a stopped group sensor.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run=edge_group, + minimum_interval_seconds=7, + default_status="stopped", +) +edge_factory_results.append(edge_health) diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs index cf9e048bb0..112ceab0af 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -24,9 +24,25 @@ fn document( pub(crate) fn decode_definitions( documents: &BTreeMap, ) -> Result { - Ok(Definitions { + let mut definitions = Definitions { jobs: document(documents, "jobs")?, groups: document(documents, "groups")?, - triggers: document(documents, "triggers")?, - }) + ..Definitions::default() + }; + for (name, value) in document::(documents, "triggers")? { + if value.get("when").is_some() { + definitions.sensors.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } else { + definitions.triggers.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } + } + Ok(definitions) } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index fda8521da8..75c383f6bc 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod documents; +pub(crate) mod operator_state; pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs new file mode 100644 index 0000000000..b07ce5e73b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs @@ -0,0 +1,81 @@ +//! Apply the operator's projection and return the before/after audit images in the same transaction. + +use serde_json::{Value, json}; + +use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::models::{ + EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerDeclaration, TriggerStatus, +}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::main::reset_sensor_cursor::reset_sensor_cursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn apply_operator_change( + transaction: ControlTransaction, + mut state: EditableTriggerState, + request: &TriggerChangeRequest, +) -> Result<(ControlTransaction, (Value, Value)), ControlError> { + let (mut transaction, cursor) = match &state.definition_json { + TriggerDeclaration::Sensor(_) => { + if matches!(request.change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "sensor cursors use storage order; use reset-cursor to replay retained events" + .into(), + )); + } + let (transaction, cursor) = read_sensor_cursor(transaction, request.trigger_id).await?; + (transaction, Some(cursor)) + } + TriggerDeclaration::Schedule(_) => (transaction, None), + }; + let mut before = + json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for}); + if let Some(cursor) = &cursor { + before["cursor"] = json!(cursor); + } + match request.change { + TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), + TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), + TriggerChange::ResetDefault => state.status_override = None, + TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), + TriggerChange::ResetCursor => state.last_tick_for = None, + } + let (status, next) = match &state.definition_json { + TriggerDeclaration::Schedule(definition) => trigger_timing( + definition, + state.started_at, + state.last_tick_for, + state.status_override, + )?, + TriggerDeclaration::Sensor(definition) => { + if matches!(request.change, TriggerChange::ResetCursor) { + transaction = + reset_sensor_cursor(transaction, (request.trigger_id, request.event_id)) + .await?; + } + let status = state + .status_override + .unwrap_or(definition.default_status.projection()); + ( + status, + (status == TriggerStatus::Running).then_some(transaction.now), + ) + } + }; + let mut after = json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for, "reason": request.reason}); + if let Some(cursor) = cursor { + after["cursor"] = json!(if matches!(request.change, TriggerChange::ResetCursor) { + Default::default() + } else { + cursor + }); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", + vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), + Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + Ok((transaction, (before, after))) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index 0977da705b..095353123a 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -4,10 +4,8 @@ use serde_json::json; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::declarations::_helpers::trigger_state::trigger_timing; -use crate::declarations::models::{ - EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerStatus, -}; +use crate::declarations::_helpers::operator_state::apply_operator_change; +use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -52,48 +50,24 @@ pub async fn change_trigger( vec![Parameter::Id(request.trigger_id)], ) .await?; - let mut state: EditableTriggerState = - decode_row(rows.into_iter().next().ok_or_else(|| { - ControlError::Invalid("trigger does not exist in the current snapshot".into()) - })?)?; - let before = json!({ - "status_override": state.status_override, - "last_tick_for": state.last_tick_for, - }); - match request.change { - TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), - TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), - TriggerChange::ResetDefault => state.status_override = None, - TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), - TriggerChange::ResetCursor => state.last_tick_for = None, - } - let (status, next) = trigger_timing( - &state.definition_json, - state.started_at, - state.last_tick_for, - state.status_override, - )?; + let state: EditableTriggerState = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + let snapshot_id = state.snapshot_id; let now = transaction.now; let (transaction, _) = query(transaction, "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; - let (transaction, _) = query(transaction, - "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", - vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), - Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + let (transaction, (before, after)) = apply_operator_change(transaction, state, request).await?; commit( append_audit( transaction, context, (request.event_id, "trigger.changed"), ( - json!({"trigger_id": request.trigger_id, "snapshot_id": state.snapshot_id}), + json!({"trigger_id": request.trigger_id, "snapshot_id": snapshot_id}), before, - json!({ - "status_override": state.status_override, - "last_tick_for": state.last_tick_for, - "reason": request.reason, - }), + after, ), ) .await?, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 9dd685ac6c..230f3c116d 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::sensors::main::publish_sensors::publish_sensors; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -19,6 +20,7 @@ pub(crate) async fn publish_definitions( (transaction, _) = query(transaction, "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + transaction = publish_sensors(transaction, snapshot, &definitions).await?; for (name, mut trigger) in definitions.triggers { let id = Uuid::new_v5( &snapshot.location_id, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index 5966cd18b3..cbe7c7a152 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -8,6 +8,7 @@ use crate::declarations::models::{Definitions, Schedule}; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::parse_cron::parse_cron; use crate::scheduler::models::CatchUp; +use crate::sensors::main::validate_sensors::validate_sensors; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; @@ -80,5 +81,6 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result {} } } + validate_sensors(&definitions)?; Ok(definitions) } diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 98f516c0d3..5231762051 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -6,6 +6,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::scheduler::models::{CatchUp, Overlap}; +use crate::sensors::models::SensorDefinition; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -90,6 +91,15 @@ pub struct Definitions { pub jobs: BTreeMap, pub groups: BTreeMap, pub triggers: BTreeMap, + pub sensors: BTreeMap, +} + +/// Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(untagged)] +pub enum TriggerDeclaration { + Schedule(TriggerDefinition), + Sensor(SensorDefinition), } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] @@ -97,7 +107,7 @@ pub struct TriggerRecord { pub trigger_id: uuid::Uuid, pub name: String, pub job_key: String, - pub definition_json: TriggerDefinition, + pub definition_json: TriggerDeclaration, pub status: TriggerStatus, pub last_tick_for: Option>, pub next_due_at: Option>, @@ -133,7 +143,7 @@ pub struct TriggerChangeRecord { #[derive(Clone, Debug, Deserialize)] pub(crate) struct EditableTriggerState { - pub(crate) definition_json: TriggerDefinition, + pub(crate) definition_json: TriggerDeclaration, pub(crate) snapshot_id: uuid::Uuid, pub(crate) started_at: DateTime, pub(crate) last_tick_for: Option>, diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 2546686e46..cc435f1701 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -8,6 +8,7 @@ pub mod declarations; pub mod platform; pub mod runs; pub mod scheduler; +pub mod sensors; pub mod snapshots; pub mod store; pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index 1757b07c66..16ded206d3 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -78,7 +78,7 @@ async fn scan_trigger_page( "SELECT t.trigger_id, t.definition_json, s.next_due_at \ FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + WHERE t.kind='schedule' AND s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ {cursor} ORDER BY t.trigger_id LIMIT 100" ), after.into_iter().map(Parameter::Id).collect(), diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs index 73879b7c63..eb5547281b 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs @@ -1,11 +1,10 @@ //! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. -use crate::platform::constants::SINK_PAGE_SIZE; -use crate::platform::models::{PlatformEventEnvelope, PlatformWebhook}; +use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformWebhook; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -17,18 +16,13 @@ pub(crate) async fn deliver_platform_events( let transaction = begin(store, TransactionMode::Read).await?; let (transaction, rows) = query( transaction, - "SELECT storage_id, event_id, occurred_at, payload FROM recall_events \ - WHERE producer='platform' AND invocation_id LIKE 'platform/%' AND storage_id > \ - (SELECT COALESCE(MAX(storage_id), 0) FROM sqb_platform_sink_deliveries WHERE sink_id=$1) \ - ORDER BY storage_id LIMIT $2", - vec![ - Parameter::Text(sink.sink_id.clone()), - Parameter::Integer(SINK_PAGE_SIZE), - ], + "SELECT COALESCE(MAX(storage_id), 0) AS cursor FROM sqb_platform_sink_deliveries WHERE sink_id=$1", + vec![Parameter::Text(sink.sink_id.clone())], ) .await?; - let events: Vec = - rows.into_iter().map(decode_row).collect::>()?; + let cursor = serde_json::from_value(rows[0]["cursor"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, events) = read_platform_events(transaction, cursor).await?; commit(transaction).await?; let count = events.len(); for event in events { diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index f405071bd8..75d8a07df7 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -3,6 +3,6 @@ /// Operator-owned delivery endpoint; never inherited by project workers or printed in logs. pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; -pub(crate) const SINK_PAGE_SIZE: i64 = 100; +pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; pub(crate) const SINK_REQUEST_TIMEOUT_SECONDS: u64 = 10; pub(crate) const SINK_POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index e9b14619b3..850f9e7c04 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -4,6 +4,7 @@ pub mod connect_location_work; pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; +pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs new file mode 100644 index 0000000000..48a38850e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs @@ -0,0 +1,22 @@ +//! One authoritative namespace filter protects every platform stream consumer from worker input. + +use crate::platform::constants::PLATFORM_EVENT_PAGE_SIZE; +use crate::platform::models::PlatformEventEnvelope; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_platform_events( + transaction: ControlTransaction, + after: i64, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,payload FROM recall_events WHERE producer='platform' \ + AND invocation_id LIKE 'platform/%' AND storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(PLATFORM_EVENT_PAGE_SIZE)]).await?; + Ok(( + transaction, + rows.into_iter().map(decode_row).collect::>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 72340bfd81..09ded290f5 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -110,6 +110,17 @@ pub(crate) struct StoredLocationWork { pub(crate) claimed_until: Option>, } +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum PlatformEventKind { + Heartbeat, + SchedulerNoLeader, + TickOverdue, + CalendarHorizonLow, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { @@ -129,6 +140,16 @@ pub enum PlatformEvent { } impl PlatformEvent { + #[must_use] + pub const fn kind(&self) -> PlatformEventKind { + match self { + Self::Heartbeat { .. } => PlatformEventKind::Heartbeat, + Self::SchedulerNoLeader => PlatformEventKind::SchedulerNoLeader, + Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, + Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, + } + } + #[must_use] pub const fn event_type(&self) -> &'static str { match self { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 4d4785b696..a2c223fbb9 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -4,75 +4,11 @@ use std::collections::BTreeSet; use uuid::Uuid; -use crate::credentials::models::AuditContext; use crate::declarations::main::frozen_definitions::frozen_definitions; -use crate::declarations::models::JobDefinition; use crate::runs::main::job_write_set::job_write_set; -use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::store::errors::ControlError; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; - -pub(crate) async fn frozen_job_request( - transaction: ControlTransaction, - context: &AuditContext, - request: &JobRunRequest, -) -> Result<(ControlTransaction, RunRequest), ControlError> { - let (transaction, jobs) = query( - transaction, - "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ - JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ - JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", - vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], - ).await?; - let row = jobs.first().ok_or_else(|| { - ControlError::Invalid("published job not found in this location and target".into()) - })?; - let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let image_id = serde_json::from_value(row["image_id"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let kind: RunKind = serde_json::from_value(row["kind"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, write_set) = - job_write_set(transaction, snapshot_id, &definition.selection).await?; - Ok(( - transaction, - RunRequest { - run_id: request.run_id, - location_id: request.location_id, - target: request.target.clone(), - snapshot_id, - image_id, - kind, - origin: if request.window.is_some() { - RunOrigin::Backfill - } else { - RunOrigin::Manual - }, - job_key: Some(request.job_key.clone()), - parent_run_id: None, - root_run_id: request.run_id, - trigger_id: None, - tick_id: None, - rerun_of_run_id: None, - requested_by: context.actor.clone(), - priority: request.priority, - selection: definition.selection, - write_set: if kind == RunKind::Group { - Default::default() - } else { - write_set - }, - settings: definition.settings, - window: request.window.clone(), - concurrency_groups: definition.concurrency_groups, - overlap: definition.overlap, - }, - )) -} +use crate::store::models::ControlTransaction; pub(crate) async fn frozen_group_children( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs new file mode 100644 index 0000000000..e91697d05d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -0,0 +1,70 @@ +//! Freeze published job inputs for manual requests and sensor evaluations alike. + +use crate::credentials::models::AuditContext; +use crate::declarations::models::JobDefinition; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_job_request( + transaction: ControlTransaction, + context: &AuditContext, + request: &JobRunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (transaction, jobs) = query( + transaction, + "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ + JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], + ).await?; + let row = jobs.first().ok_or_else(|| { + ControlError::Invalid("published job not found in this location and target".into()) + })?; + let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let image_id = serde_json::from_value(row["image_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let kind: RunKind = serde_json::from_value(row["kind"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = + job_write_set(transaction, snapshot_id, &definition.selection).await?; + Ok(( + transaction, + RunRequest { + run_id: request.run_id, + location_id: request.location_id, + target: request.target.clone(), + snapshot_id, + image_id, + kind, + origin: if request.window.is_some() { + RunOrigin::Backfill + } else { + RunOrigin::Manual + }, + job_key: Some(request.job_key.clone()), + parent_run_id: None, + root_run_id: request.run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: context.actor.clone(), + priority: request.priority, + selection: definition.selection, + write_set: if kind == RunKind::Group { + Default::default() + } else { + write_set + }, + settings: definition.settings, + window: request.window.clone(), + concurrency_groups: definition.concurrency_groups, + overlap: definition.overlap, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 6359a19779..6027f5ea35 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,6 +1,7 @@ pub mod advance_groups; pub mod change_run; pub mod claim_next_run; +pub(crate) mod frozen_job_request; pub mod heartbeat_run; pub(crate) mod job_write_set; pub mod list_run_claims; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs index 5df18cde15..8593e77c41 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::runs::_helpers::requests::job::frozen_job_request; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::main::frozen_job_request::frozen_job_request; use crate::runs::models::{JobRunRequest, RunWindow}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index f2edb08a87..ba0d626c09 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -2,6 +2,7 @@ use uuid::Uuid; +use crate::scheduler::_helpers::drain_sensors::drain_sensors; use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; @@ -21,7 +22,7 @@ pub(crate) async fn drain_due_triggers( let (transaction, rows) = query(transaction, "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + WHERE t.kind='schedule' AND s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; commit(transaction).await?; for row in rows { @@ -33,5 +34,7 @@ pub(crate) async fn drain_due_triggers( } lease = renew_lease(store, &lease).await?; } + let (lease, sensor_evaluations) = drain_sensors(store, lease).await?; + cycle.evaluated += sensor_evaluations; Ok((lease, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs new file mode 100644 index 0000000000..2456383857 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs @@ -0,0 +1,36 @@ +//! The scheduler drains built-in sensors without importing project code. + +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_sensors( + store: &ControlStore, + mut lease: SchedulerLease, +) -> Result<(SchedulerLease, usize), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.kind='sensor' AND s.status='running' \ + AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at,t.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + let mut evaluated = 0; + for row in rows { + let trigger_id = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(request) = begin_sensor_evaluation(store, &lease, trigger_id).await? { + complete_builtin_sensor(store, &lease, &request).await?; + evaluated += 1; + } + lease = renew_lease(store, &lease).await?; + } + Ok((lease, evaluated)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs index 5102baaac9..c007a30d78 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod calendar; pub(crate) mod drain_due_triggers; +pub(crate) mod drain_sensors; pub(crate) mod expansion; pub(crate) mod persistence; pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs new file mode 100644 index 0000000000..310fde7a9b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs @@ -0,0 +1,65 @@ +//! Indexed reads recover attempts and schedule projections without mutable phase transitions. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::sensors::models::{DueSensor, SensorEvaluationFact}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id,t.location_id,t.target,t.snapshot_id,t.definition_json,s.status \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.trigger_id=$1 AND t.kind='sensor' \ + AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +pub(crate) async fn last_attempt( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option<(SensorEvaluationFact, bool)>), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT e.fact_json, CASE WHEN EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events terminal \ + WHERE terminal.evaluation_id=e.evaluation_id AND terminal.phase<>'requested') THEN 1 ELSE 0 END AS terminal \ + FROM sqb_sensor_evaluation_events e WHERE e.trigger_id=$1 AND e.phase='requested' ORDER BY e.storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let last = rows + .first() + .map(|row| { + let fact = serde_json::from_value(row["fact_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok::<_, ControlError>((fact, row["terminal"].as_i64() == Some(1))) + }) + .transpose()?; + Ok((transaction, last)) +} + +pub(crate) async fn defer_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, + next: DateTime, +) -> Result { + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,updated_at=$2 WHERE trigger_id=$3", + vec![ + Parameter::Time(next), + Parameter::Time(now), + Parameter::Id(trigger_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs new file mode 100644 index 0000000000..c94e5577c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -0,0 +1,65 @@ +//! Attempts and cursor commits use Recall's serialized commit-order sequence. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::models::{ + SensorCursor, SensorEvaluationFact, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn append_evaluation( + transaction: ControlTransaction, + request: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(ControlTransaction, (Uuid, i64)), ControlError> { + let phase = outcome.phase(); + let event_id = Uuid::new_v5(&request.evaluation_id, phase.as_bytes()); + let occurred_at = transaction.now; + let fact = SensorEvaluationFact { + request: request.clone(), + outcome, + }; + let payload = json!(fact); + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("sensor/{}", request.trigger_id), + event_type: "sensor_evaluation", + producer: "scheduler", + occurred_at, + run_id: None, + payload: payload.clone(), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "sensor evaluation conflicts with a durable fact".into(), + )); + } + let storage_id = appended + .storage_id + .ok_or_else(|| ControlError::Invalid("sensor evaluation has no storage cursor".into()))?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_evaluation_events (event_id,storage_id,evaluation_id,trigger_id,phase,occurred_at,fact_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(request.evaluation_id), + Parameter::Id(request.trigger_id), Parameter::Text(phase.into()), Parameter::Time(occurred_at), Parameter::Json(payload)]).await?; + Ok((transaction, (event_id, storage_id))) +} + +pub(crate) async fn commit_cursor( + transaction: ControlTransaction, + (trigger_id, event_id, storage_id): (Uuid, Uuid, i64), + cursor: &SensorCursor, +) -> Result { + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_cursor_commits (event_id,storage_id,trigger_id,cursor_json) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(trigger_id), Parameter::Json(json!(cursor))]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs new file mode 100644 index 0000000000..498b74207a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod attempts; +pub(crate) mod facts; +pub(crate) mod platform; +pub(crate) mod requests; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs new file mode 100644 index 0000000000..8215ce4379 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -0,0 +1,34 @@ +//! One bounded platform page yields matched run requests and a storage-order cursor. + +use uuid::Uuid; + +use crate::platform::main::read_platform_events::read_platform_events; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_platform_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, events) = + read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; + let cursor = SensorCursor { + event_storage_id: events + .last() + .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), + }; + let SensorCondition::PlatformEvent { kinds } = &evaluation.definition.when; + let mut run_ids: Vec = Vec::new(); + for event in events + .iter() + .filter(|event| kinds.contains(&event.payload.platform.kind())) + { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs new file mode 100644 index 0000000000..237959031b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -0,0 +1,62 @@ +//! A sensor run key gates queue publication within the cursor/outcome transaction. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{JobRunRequest, RunOrigin}; +use crate::sensors::models::SensorEvaluationRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_sensor_run( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + run_key: &str, +) -> Result<(ControlTransaction, Uuid), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT run_id FROM sqb_run_requests WHERE trigger_id=$1 AND run_key=$2", + vec![ + Parameter::Id(evaluation.trigger_id), + Parameter::Text(run_key.into()), + ], + ) + .await?; + if let Some(row) = rows.into_iter().next() { + return Ok(( + transaction, + serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + )); + } + let run_id = Uuid::new_v5(&evaluation.trigger_id, run_key.as_bytes()); + let context = AuditContext { + actor: "system:sensor".into(), + granting_role: "system:scheduler".into(), + request_id: evaluation.evaluation_id, + ip_address: None, + }; + let authored = JobRunRequest { + run_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + job_key: evaluation.definition.job_key.clone(), + reason: "Sensor condition matched".into(), + priority: 0, + window: None, + }; + let (transaction, mut frozen) = frozen_job_request(transaction, &context, &authored).await?; + if frozen.snapshot_id != evaluation.snapshot_id { + return Err(ControlError::Fenced); + } + frozen.origin = RunOrigin::Sensor; + frozen.trigger_id = Some(evaluation.trigger_id); + let transaction = queue_run(transaction, &frozen).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_requests (trigger_id,run_key,evaluation_id,run_id) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(evaluation.trigger_id), Parameter::Text(run_key.into()), Parameter::Id(evaluation.evaluation_id), Parameter::Id(run_id)]).await?; + Ok((transaction, run_id)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs new file mode 100644 index 0000000000..45b82e4cbb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -0,0 +1,80 @@ +//! Commit the requested phase before evaluation, so crashes count as attempts. + +use chrono::TimeDelta; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::append_evaluation; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn begin_sensor_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, sensor) = read_sensor(transaction, trigger_id).await?; + let Some(sensor) = sensor else { + return Ok(None); + }; + if sensor.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let (transaction, cursor) = read_sensor_cursor(transaction, trigger_id).await?; + let (mut transaction, previous) = last_attempt(transaction, trigger_id).await?; + if let Some((previous, terminal)) = previous { + if !terminal { + if previous.request.epoch == lease.epoch + && previous.request.snapshot_id == sensor.snapshot_id + && previous.request.cursor == cursor + { + commit(transaction).await?; + return Ok(Some(previous.request)); + } + (transaction, _) = append_evaluation( + transaction, + &previous.request, + SensorEvaluationOutcome::Abandoned, + ) + .await?; + } + let next = previous.request.requested_at + + TimeDelta::seconds(i64::from( + sensor.definition_json.minimum_interval_seconds.get(), + )); + if next > now { + commit(defer_sensor(transaction, trigger_id, next).await?).await?; + return Ok(None); + } + } + let definition_fingerprint = sha256_hex( + &serde_json::to_vec(&sensor.definition_json) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let request = SensorEvaluationRequest { + evaluation_id: Uuid::now_v7(), + trigger_id: sensor.trigger_id, + location_id: sensor.location_id, + target: sensor.target, + snapshot_id: sensor.snapshot_id, + definition: sensor.definition_json, + definition_fingerprint, + cursor, + epoch: lease.epoch, + requested_at: now, + }; + let (transaction, _) = + append_evaluation(transaction, &request, SensorEvaluationOutcome::Requested).await?; + commit(check_lease(transaction, lease).await?).await?; + Ok(Some(request)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs new file mode 100644 index 0000000000..5fcf9286c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -0,0 +1,73 @@ +//! Cursor, outcome, deduplicated requests and queued runs commit together. + +use chrono::TimeDelta; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; +use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn complete_builtin_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + if evaluation.epoch != lease.epoch { + return Err(ControlError::Fenced); + } + let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; + let Some((previous, terminal)) = previous else { + return Err(ControlError::Fenced); + }; + if previous.request != *evaluation { + return Err(ControlError::Fenced); + } + if terminal { + commit(transaction).await?; + return Ok(0); + } + let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; + let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; + if sensor.is_none_or(|sensor| { + sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id + }) || cursor != evaluation.cursor + { + let (transaction, _) = + append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; + commit(transaction).await?; + return Ok(0); + } + let (transaction, (cursor, run_ids)) = consume_platform_events(transaction, evaluation).await?; + let count = run_ids.len(); + let (transaction, (event_id, storage_id)) = append_evaluation( + transaction, + evaluation, + SensorEvaluationOutcome::Completed { + cursor: cursor.clone(), + run_ids, + }, + ) + .await?; + let transaction = commit_cursor( + transaction, + (evaluation.trigger_id, event_id, storage_id), + &cursor, + ) + .await?; + let next = evaluation.requested_at + + TimeDelta::seconds(i64::from( + evaluation.definition.minimum_interval_seconds.get(), + )); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs new file mode 100644 index 0000000000..a57c8e3058 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -0,0 +1,6 @@ +pub mod begin_sensor_evaluation; +pub mod complete_builtin_sensor; +pub(crate) mod publish_sensors; +pub(crate) mod read_sensor_cursor; +pub(crate) mod reset_sensor_cursor; +pub(crate) mod validate_sensors; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs new file mode 100644 index 0000000000..3f2029c481 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs @@ -0,0 +1,37 @@ +//! Sensor publication keeps operator overrides and immutable cursor history across deployments. + +use serde_json::json; +use uuid::Uuid; + +use crate::declarations::models::Definitions; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_sensors( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for (name, sensor) in &definitions.sensors { + let trigger_id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "sensor", name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id,location_id,target,kind,name,job_key,definition_json,snapshot_id) \ + VALUES ($1,$2,$3,'sensor',$4,$5,$6,$7) ON CONFLICT (trigger_id) DO UPDATE \ + SET job_key=excluded.job_key,definition_json=excluded.definition_json,snapshot_id=excluded.snapshot_id,removed_in_snapshot_id=NULL", + vec![Parameter::Id(trigger_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), + Parameter::Text(name.clone()), Parameter::Text(sensor.job_key.clone()), Parameter::Json(json!(sensor)), Parameter::Id(snapshot.snapshot_id)]).await?; + let status = sensor.default_status.projection(); + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id,status,started_at,next_due_at,updated_at) VALUES ($1,$2,$3,$3,$3) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=COALESCE(sqb_trigger_state.status_override,excluded.status), \ + next_due_at=excluded.next_due_at,updated_at=excluded.updated_at", + vec![Parameter::Id(trigger_id), Parameter::Text(status.as_str().into()), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs new file mode 100644 index 0000000000..ab2c060dd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs @@ -0,0 +1,24 @@ +//! Operator reads and evaluations share the latest immutable cursor commit. + +use uuid::Uuid; + +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor_cursor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, SensorCursor), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT cursor_json FROM sqb_sensor_cursor_commits WHERE trigger_id=$1 ORDER BY storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let cursor = rows + .first() + .map(|row| serde_json::from_value(row["cursor_json"].clone())) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))? + .unwrap_or_default(); + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs new file mode 100644 index 0000000000..e447028c30 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs @@ -0,0 +1,41 @@ +//! Operator reset appends a cursor fact without mutating previous commits or run keys. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::_helpers::facts::commit_cursor; +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; + +pub(crate) async fn reset_sensor_cursor( + transaction: ControlTransaction, + (trigger_id, request_id): (Uuid, Uuid), +) -> Result { + let cursor = SensorCursor::default(); + let event_id = Uuid::new_v5(&request_id, b"sensor-cursor-reset"); + let occurred_at = transaction.now; + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("sensor/{trigger_id}"), + event_type: "sensor_cursor_reset", + producer: "scheduler", + occurred_at, + run_id: None, + payload: json!({"trigger_id": trigger_id, "cursor": cursor}), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "sensor cursor reset conflicts with a durable fact".into(), + )); + } + let storage_id = appended + .storage_id + .ok_or_else(|| ControlError::Invalid("sensor cursor reset has no storage cursor".into()))?; + commit_cursor(transaction, (trigger_id, event_id, storage_id), &cursor).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs new file mode 100644 index 0000000000..06cd779823 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -0,0 +1,22 @@ +//! Native publication validates every sensor before any snapshot pointer moves. + +use crate::declarations::models::Definitions; +use crate::sensors::models::SensorCondition; +use crate::store::errors::ControlError; + +pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlError> { + for (name, sensor) in &definitions.sensors { + let SensorCondition::PlatformEvent { kinds } = &sensor.when; + if name.trim().is_empty() + || sensor.description.trim().is_empty() + || kinds.is_empty() + || (!definitions.jobs.contains_key(&sensor.job_key) + && !definitions.groups.contains_key(&sensor.job_key)) + { + return Err(ControlError::Invalid(format!( + "sensor {name} needs a description, a known job and nonempty event kinds" + ))); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs new file mode 100644 index 0000000000..ca1039eddc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -0,0 +1,9 @@ +//! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/sensors/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs new file mode 100644 index 0000000000..f39cf43d2f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -0,0 +1,114 @@ +//! Published conditions and immutable attempts share one sensor abstraction. + +use std::collections::BTreeSet; +use std::num::NonZeroU32; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::platform::models::PlatformEventKind; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SensorStatus { + #[default] + Running, + Stopped, +} + +impl SensorStatus { + #[must_use] + pub const fn projection(self) -> TriggerStatus { + match self { + Self::Running => TriggerStatus::Running, + Self::Stopped => TriggerStatus::Paused, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorCondition { + PlatformEvent { kinds: BTreeSet }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SensorDefinition { + pub description: String, + pub job_key: String, + pub when: SensorCondition, + #[schema(value_type = u32, minimum = 1)] + pub minimum_interval_seconds: NonZeroU32, + #[serde(default)] + pub default_status: SensorStatus, +} + +/// Storage order is authoritative even when a fact's occurrence time predates earlier pages. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorCursor { + pub event_storage_id: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationRequest { + pub evaluation_id: Uuid, + pub trigger_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub definition: SensorDefinition, + pub definition_fingerprint: String, + pub cursor: SensorCursor, + pub epoch: i64, + pub requested_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorEvaluationOutcome { + Requested, + Completed { + cursor: SensorCursor, + run_ids: Vec, + }, + Failed { + code: String, + }, + TimedOut, + Abandoned, +} + +impl SensorEvaluationOutcome { + #[must_use] + pub const fn phase(&self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Completed { .. } => "completed", + Self::Failed { .. } => "failed", + Self::TimedOut => "timed_out", + Self::Abandoned => "abandoned", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationFact { + pub request: SensorEvaluationRequest, + pub outcome: SensorEvaluationOutcome, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct DueSensor { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) snapshot_id: Uuid, + pub(crate) definition_json: SensorDefinition, + pub(crate) status: TriggerStatus, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index fd922e7d2f..5ae6a26780 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -10,6 +10,7 @@ pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); +pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 56f2194ffb..dcc49f4793 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, render, verify_migrations, + LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, SENSORS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -31,6 +31,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("location_instances", LOCATION_INSTANCES), ("location_work", LOCATION_WORK), ("platform_sink", PLATFORM_SINK), + ("sensors", SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/sensors.sql b/crates/sqlbuild-orchestration/src/store/sensors.sql new file mode 100644 index 0000000000..9e1ab95213 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/sensors.sql @@ -0,0 +1,25 @@ +CREATE TABLE sqb_sensor_evaluation_events ( + event_id {uuid} PRIMARY KEY, + storage_id BIGINT NOT NULL UNIQUE, + evaluation_id {uuid} NOT NULL, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at {time} NOT NULL, + fact_json {json} NOT NULL, + UNIQUE (evaluation_id, phase) +); +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, storage_id); +CREATE TABLE sqb_sensor_cursor_commits ( + event_id {uuid} PRIMARY KEY, + storage_id BIGINT NOT NULL UNIQUE, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + cursor_json {json} NOT NULL +); +CREATE INDEX sqb_sensor_cursor_history ON sqb_sensor_cursor_commits(trigger_id, storage_id); +CREATE TABLE sqb_run_requests ( + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id {uuid} NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 7d77cdf9fa..4af6d90bef 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 4c388072ab..41610c718b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -308,7 +308,7 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur .expect("refreshed trigger") .remove(0); assert_eq!( - serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], json!([ "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", @@ -356,7 +356,7 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur .expect("drained trigger") .remove(0); assert_eq!( - serde_json::to_value(&trigger.definition_json.schedule).expect("calendar")["times"], + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], json!(["2032-01-01T00:00:00Z"]) ); assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs new file mode 100644 index 0000000000..7aea4d6287 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -0,0 +1,114 @@ +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::platform::models::PlatformEvent; +use crate::sensors::models::SensorEvaluationFact; +use crate::snapshots::models::SnapshotInput; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlEvent, ControlStore, TransactionMode}; + +use crate::runs::tests::helpers::snapshot as base_snapshot; +use crate::store::tests::helpers::read_recall_events; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let mut input = base_snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({ + "orders": {"description": "Refresh orders", "selection": ["orders"], "overlap": "allow"} + }), + ); + input.documents.insert( + "triggers".into(), + json!({ + "health_orders": { + "description": "Refresh reporting on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 1 + } + }), + ); + input +} + +pub(crate) async fn append_events( + store: &ControlStore, + indices: std::ops::Range, + occurred_at: DateTime, +) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("event transaction"); + let payloads = [ + PlatformEvent::TickOverdue { + trigger_id: Uuid::from_u128(7), + scheduled_for: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(8), + }, + ]; + for index in indices { + let platform = &payloads[(index % 2) as usize]; + let (next, appended) = append_control_event( + transaction, + ControlEvent { + event_id: Uuid::from_u128(1000 + index), + invocation_id: "platform/test-health".into(), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload: json!({"platform": platform}), + }, + ) + .await + .expect("append platform observation"); + assert!(appended.inserted); + transaction = next; + } + commit(transaction).await.expect("commit observations"); +} + +pub(crate) async fn evaluations(store: &ControlStore) -> Vec { + read_recall_events(store, 0, 1000) + .await + .expect("read durable history") + .into_iter() + .filter(|row| row["event_type"] == "sensor_evaluation") + .map(|row| serde_json::from_value(row["payload"].clone()).expect("typed attempt fact")) + .collect() +} + +pub(crate) async fn change(store: &ControlStore, trigger_id: Uuid, change: TriggerChange) { + let event_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: event_id, + ip_address: None, + }; + let request = TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason: "Test operator change".into(), + change, + }; + assert!( + change_trigger(store, &context, &request) + .await + .expect("apply operator fact") + ); + assert!( + !change_trigger(store, &context, &request) + .await + .expect("retry operator fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs new file mode 100644 index 0000000000..04f4485f1e --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -0,0 +1,10 @@ +//! Real stores prove atomic sensor consumption and recovery across scheduler ownership changes. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_delivery.rs"] +mod test_delivery; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs new file mode 100644 index 0000000000..de5581844b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs @@ -0,0 +1,183 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_paged_platform_events_when_sensor_publication_fails_then_retry_is_atomic_and_reset_deduplicates() + { + let test_cases = [ + SensorTestCase { + description: "SQLite sensor delivery", + backend: Backend::Sqlite, + expected_runs: 54, + }, + SensorTestCase { + description: "PostgreSQL sensor delivery", + backend: Backend::Postgres, + expected_runs: 54, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let now = Utc::now(); + append_events(store, 0..105, now).await; + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + let replica = second_store(store).await; + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(requested.clone()) + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + "sqb_sensor_cursor_commits", + ] { + let point = TransactionFault { + description: "fail sensor transaction", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err(), + "{table}" + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rolled back runs") + .is_empty(), + "{table}" + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "only requested survives {table}" + ); + } + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry first page"), + 50 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("retry completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next page") + .expect("due"); + assert!(second.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &second) + .await + .expect("remaining page"), + 3 + ); + // Committed later, but its occurrence precedes the entire first page. + append_events(store, 106..107, now - TimeDelta::days(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let late = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("late request") + .expect("due"); + assert!(late.cursor.event_storage_id > second.cursor.event_storage_id); + assert_eq!( + complete_builtin_sensor(store, &lease, &late) + .await + .expect("late arrival"), + 1 + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + assert_eq!(replay.cursor.event_storage_id, 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay retained first page"), + 50 + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("queued runs"); + assert_eq!( + runs.len(), + test_case.expected_runs, + "{}: reset never requeues an existing run key", + test_case.description + ); + assert!( + runs.iter() + .all(|run| run.selection_json == BTreeSet::from(["orders".into()])) + ); + let facts = evaluations(store).await; + assert_eq!(facts.len(), 8); + assert!(facts.iter().all(|fact| matches!( + fact.outcome, + SensorEvaluationOutcome::Requested | SensorEvaluationOutcome::Completed { .. } + ))); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs new file mode 100644 index 0000000000..426fc861a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs @@ -0,0 +1,218 @@ +use std::time::Duration; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerStatus}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_crashed_sensor_when_leadership_or_snapshot_changes_then_stale_results_cannot_queue_runs() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite sensor recovery", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL sensor recovery", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + append_events(store, 0..1, chrono::Utc::now()).await; + let first = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let pending = begin_sensor_evaluation(store, &first, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + expire_lease(store).await; + let replica = second_store(store).await; + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + complete_builtin_sensor(store, &first, &pending).await, + Err(ControlError::Fenced) + )); + assert!( + begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned attempt") + .is_none(), + "a crash still counts toward the minimum interval" + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("abandoned fact") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let requested = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("new request") + .expect("due"); + assert_ne!(pending.evaluation_id, requested.evaluation_id); + let mut forged = requested.clone(); + forged.cursor.event_storage_id = 999; + assert!(matches!( + complete_builtin_sensor(store, &successor, &forged).await, + Err(ControlError::Fenced) + )); + input.generation += 1; + publish_snapshot(store, &input).await.expect("redeploy"); + assert_eq!( + complete_builtin_sensor(store, &successor, &requested) + .await + .expect("abandon stale snapshot"), + 0 + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale runs") + .is_empty() + ); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + input.generation += 1; + publish_snapshot(store, &input) + .await + .expect("redeploy paused sensor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("preserved override")[0] + .status, + TriggerStatus::Paused + ); + assert!( + begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("paused") + .is_none() + ); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let current = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("code default") + .expect("running"); + assert_eq!( + complete_builtin_sensor(store, &successor, ¤t) + .await + .expect("catch up retained health"), + test_case.expected_runs, + "{}", + test_case.description + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_reset = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before reset") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_reset) + .await + .expect("reset fences old cursor"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("reset abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_pause = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before pause") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_pause) + .await + .expect("pause fences pending result"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("pause abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let delayed = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request delayed evaluation") + .expect("due"); + tokio::time::sleep(Duration::from_secs(1)).await; + complete_builtin_sensor(store, &successor, &delayed) + .await + .expect("complete delayed evaluation"); + let immediate = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("interval measured from start, not completion") + .expect("already due"); + assert_ne!(immediate.evaluation_id, delayed.evaluation_id); + complete_builtin_sensor(store, &successor, &immediate) + .await + .expect("complete next attempt"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("deduplicated runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs new file mode 100644 index 0000000000..b762f2f4c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -0,0 +1,7 @@ +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct SensorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 7b2283fbd1..5507104d12 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_platform_sink_deliveries; \ + "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ + DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 9706fcc399..be7fa9ac3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -16,16 +16,16 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDefinition, + Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDeclaration, TriggerDefinition, }; use sqlbuild_orchestration::scheduler::models::Overlap; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, module_jobs, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, module_jobs, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - compile_calendars, compile_cron, + compile_calendars, compile_cron, compile_sensors, }; /// Dedicated declaration modules are imported only in the code-location interpreter. @@ -78,11 +78,13 @@ pub(crate) fn compile_declarations( let mut authored_groups: Vec = Vec::new(); let mut authored_jobs: Vec = Vec::new(); let mut authored_calendars: Vec> = Vec::new(); + let mut authored_sensors: Vec = Vec::new(); for declaration in declarations { match declaration { AuthoredDeclaration::Job(job) => authored_jobs.push(job), AuthoredDeclaration::Group(group) => authored_groups.push(group), AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), + AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), } } let mut triggers: BTreeMap = BTreeMap::new(); @@ -198,16 +200,31 @@ pub(crate) fn compile_declarations( jobs, groups, triggers, - } = compile_calendars( - py, - (project_dir, &input.target), - authored_calendars, - Definitions { - jobs, - groups, - triggers, - }, + sensors, + } = compile_sensors( + &input.target, + authored_sensors, + compile_calendars( + py, + (project_dir, &input.target), + authored_calendars, + Definitions { + jobs, + groups, + triggers, + ..Definitions::default() + }, + )?, )?; + let triggers: BTreeMap<_, _> = triggers + .into_iter() + .map(|(name, trigger)| (name, TriggerDeclaration::Schedule(trigger))) + .chain( + sensors + .into_iter() + .map(|(name, sensor)| (name, TriggerDeclaration::Sensor(sensor))), + ) + .collect(); for (key, document) in [ ("jobs", serde_json::to_value(jobs)), ("groups", serde_json::to_value(groups)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index d2efd5db0b..84031a99e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -10,7 +10,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::git_revisions::main::head_commit::head_commit; -use sqlbuild_orchestration::declarations::models::{Schedule, TriggerDefinition}; +use sqlbuild_orchestration::declarations::models::{ + Schedule, TriggerDeclaration, TriggerDefinition, +}; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; @@ -189,7 +191,7 @@ pub(super) fn publish_location( .extract::()?; let input = metadata(py, arguments, &directory)?; let (snapshot, _) = compile_location(py, arguments, &input)?; - let triggers: BTreeMap = serde_json::from_value( + let triggers: BTreeMap = serde_json::from_value( snapshot .input .documents @@ -200,9 +202,12 @@ pub(super) fn publish_location( .map_err(|error| PyValueError::new_err(error.to_string()))?; let refresh_at = triggers .values() - .filter_map(|trigger| match &trigger.schedule { - Schedule::Calendar { refresh_at, .. } => Some(*refresh_at), - Schedule::Cron { .. } => None, + .filter_map(|trigger| match trigger { + TriggerDeclaration::Schedule(TriggerDefinition { + schedule: Schedule::Calendar { refresh_at, .. }, + .. + }) => Some(*refresh_at), + TriggerDeclaration::Schedule(_) | TriggerDeclaration::Sensor(_) => None, }) .min(); status(py, "Publishing location snapshot...")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 1b04b08ad8..24a1f9f20e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -46,10 +46,29 @@ pub(crate) struct AuthoredGroup { pub(crate) targets: Vec, } +#[derive(FromPyObject)] +pub(crate) struct AuthoredPlatformCondition { + pub(crate) kind: String, + pub(crate) kinds: Vec, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredSensor { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) when: AuthoredPlatformCondition, + pub(crate) run: AuthoredJobReference, + #[pyo3(from_py_with = authored_setting)] + pub(crate) minimum_interval_seconds: Value, + pub(crate) default_status: String, + pub(crate) targets: Vec, +} + pub(crate) enum AuthoredDeclaration { Job(AuthoredJob), Group(AuthoredGroup), Calendar(Py), + Sensor(AuthoredSensor), } /// JSON authoring values cross once; the interpreter rejects circular user containers safely. @@ -77,6 +96,7 @@ pub(crate) fn module_jobs<'py>( let calendar_class = py .import("sqlbuild.orchestration")? .getattr("CalendarSchedule")?; + let sensor_class = py.import("sqlbuild.orchestration")?.getattr("Sensor")?; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); @@ -88,6 +108,7 @@ pub(crate) fn module_jobs<'py>( if value.is_instance(&job_class)? || value.is_instance(&group_class)? || value.is_instance(&calendar_class)? + || value.is_instance(&sensor_class)? { jobs.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { @@ -106,6 +127,7 @@ pub(crate) fn collect_declarations<'py>( let calendar = py .import("sqlbuild.orchestration")? .getattr("CalendarSchedule")?; + let sensor = py.import("sqlbuild.orchestration")?.getattr("Sensor")?; let mut roots: HashSet = HashSet::new(); for declaration in &declarations { if !roots.insert(declaration.as_ptr() as usize) { @@ -125,7 +147,16 @@ pub(crate) fn collect_declarations<'py>( if !seen.insert(declaration.as_ptr() as usize) { continue; } - let authored = if declaration.is_instance(&calendar)? { + let authored = if declaration.is_instance(&sensor)? { + let target = declaration.getattr("run")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "sensor run must be a Job or JobGroup object", + )); + } + pending.push(target); + AuthoredDeclaration::Sensor(declaration.extract()?) + } else if declaration.is_instance(&calendar)? { let target = declaration.getattr("job")?; if !target.is_instance(&job)? && !target.is_instance(&group)? { return Err(PyTypeError::new_err( @@ -159,7 +190,7 @@ pub(crate) fn collect_declarations<'py>( AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( - "snapshot declarations must be Job, JobGroup or CalendarSchedule objects", + "snapshot declarations must be Job, JobGroup, CalendarSchedule or Sensor objects", )); } else { AuthoredDeclaration::Job(declaration.extract()?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index c5366827ed..b787901fc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -9,18 +9,21 @@ use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use serde_json::Value; +use serde_json::{Value, json}; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; +use sqlbuild_orchestration::sensors::models::SensorDefinition; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::native_host::constants::CALENDAR_WINDOW_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; -use crate::bindings::_helpers::native_host::orchestration_declarations::authored_setting; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + AuthoredSensor, authored_setting, +}; #[derive(FromPyObject)] struct AuthoredControls { @@ -243,3 +246,32 @@ pub(crate) fn compile_calendars( Ok(definitions) }) } + +pub(crate) fn compile_sensors( + target: &str, + authored: Vec, + mut definitions: Definitions, +) -> PyResult { + let mut names: BTreeSet = BTreeSet::new(); + for sensor in authored { + if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { + return Err(PyValueError::new_err(format!( + "duplicate trigger {}", + sensor.name + ))); + } + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { + continue; + } + let definition: SensorDefinition = serde_json::from_value(json!({ + "description": sensor.description, + "job_key": sensor.run.name, + "when": {"kind": sensor.when.kind, "kinds": sensor.when.kinds}, + "minimum_interval_seconds": sensor.minimum_interval_seconds, + "default_status": sensor.default_status, + })) + .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; + definitions.sensors.insert(sensor.name, definition); + } + Ok(definitions) +} diff --git a/python-boundary.toml b/python-boundary.toml index b02d91a88d..c5d7d33fc7 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -22,6 +22,11 @@ path = "orchestration_declarations/models.py" rule = 3 note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." +[[stays]] +path = "orchestration_declarations/types.py" +rule = 3 +note = "Public PlatformEventKind enum only, placed in types.py to satisfy enum ownership; no Python event processing or implementation allowance." + [[stays]] path = "orchestration_declarations/main/compile_snapshot.py" rule = 3 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b235862344..84509d6ae7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1739,7 +1739,7 @@ class NativeCompiledAudit: def meta(self) -> dict[str, object] | None: ... class NativeOrchestrationSnapshot: - """A compiled data-only publication retained natively until transport.""" + """Compiled jobs, schedules and sensors retained as data until native publication.""" def to_json(self) -> str: ... def publish(self, server_url: str, bearer: str) -> str: ... diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 997e6d0181..0920f49d17 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -6,22 +6,32 @@ CalendarWindow, Job, JobGroup, + PlatformEventCondition, Schedule, + Sensor, ) from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group +from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event from sqlbuild.orchestration_declarations.models import Schedule as schedule +from sqlbuild.orchestration_declarations.models import Sensor as sensor +from sqlbuild.orchestration_declarations.types import PlatformEventKind __all__ = ( "CalendarSchedule", "CalendarWindow", "Job", "JobGroup", + "PlatformEventCondition", + "PlatformEventKind", "Schedule", + "Sensor", "calendar_schedule", "compile_snapshot", "job", "job_group", + "platform_event", "schedule", + "sensor", ) diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index b9e09facb3..cc7f6c83f2 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,7 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup +from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup, Sensor def compile_snapshot( @@ -15,7 +15,7 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, - declarations: Sequence[Job | JobGroup | CalendarSchedule] | None = None, + declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index b9617e4ffe..8086e34bf6 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -7,6 +7,8 @@ from datetime import datetime from typing import Literal +from sqlbuild.orchestration_declarations.types import PlatformEventKind + @dataclass(frozen=True, kw_only=True) class Schedule: @@ -78,3 +80,24 @@ class CalendarSchedule: def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class PlatformEventCondition: + """Consume matching server observations in durable storage order.""" + + kinds: Sequence[PlatformEventKind] + kind: Literal["platform_event"] = field(default="platform_event", init=False) + + +@dataclass(frozen=True, eq=False) +class Sensor: + """Start a job for each matching event; event identity deduplicates replayed requests.""" + + name: str + description: str = field(kw_only=True) + when: PlatformEventCondition = field(kw_only=True) + run: Job | JobGroup = field(kw_only=True) + minimum_interval_seconds: int = field(default=30, kw_only=True) + default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py new file mode 100644 index 0000000000..c41199fb86 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -0,0 +1,12 @@ +"""Public enum vocabulary only; event processing belongs to Rust.""" + +from enum import StrEnum + + +class PlatformEventKind(StrEnum): + """Closed vocabulary of durable platform observations.""" + + HEARTBEAT = "heartbeat" + SCHEDULER_NO_LEADER = "scheduler_no_leader" + TICK_OVERDUE = "tick_overdue" + CALENDAR_HORIZON_LOW = "calendar_horizon_low" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 293ae24452..87a99a43a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -253,6 +253,8 @@ class SnapshotDiscoveryTestCase: expected_jobs: tuple[str, ...] declarations: tuple[()] | None = None expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) + expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) + expected_schedules: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 67d5f94e18..ee942f38ab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -99,6 +99,7 @@ class WorkerExecutionTestCase: expected_status: str expected_rows: list[tuple[int]] expected_node_count: int = 1 + expected_origin: Literal["schedule", "sensor"] = "schedule" files: dict[str, str] = field(default_factory=dict) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index bcc8855eca..ae3edf17d5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -375,6 +375,9 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - assert nodes[0]["started_at"] is not None assert nodes[0]["ended_at"] is not None with sqlite3.connect(control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (run["run_id"],) + ).fetchone() == (test_case.expected_origin,) assert control.execute( "SELECT count(*) FROM recall_events " "WHERE invocation_id=? AND event_type='run_launched'", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index f15bea15b5..7eb241a5fe 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -143,6 +143,9 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" ), + "orchestration/platform_orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text(), "providers/fulfillment.py": ( "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta\n" "from sqlbuild.provider.classes.provider import Provider\n" @@ -184,10 +187,13 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr second: dict[str, Any] = wait_for_generation(api, "development", 2, process, log) assert first["nodes"] == second["nodes"] assert second["changes"] == [] - assert first["documents"]["triggers"].keys() == {"fulfillment_calendar"} + assert first["documents"]["triggers"].keys() == { + "fulfillment_calendar", + "overdue_orders", + } with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: stored, next_due = control.execute( - "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id)" + "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" ).fetchone() original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ "schedule" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 31caf9be2d..0c49401511 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -121,11 +121,20 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical ) with urlopen(request, timeout=5) as response: triggers: list[dict[str, Any]] = json.load(response) - assert len(triggers) == 1 - assert triggers[0]["status"] == "paused" - assert triggers[0]["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" - assert triggers[0]["definition_json"]["default_status"] == "paused" - assert triggers[0]["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} + assert len(triggers) == 2 + assert set(by_name) == {"edge_snapshot_orders", "edge_platform_orders"} + scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] + assert scheduled["status"] == "paused" + assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" + assert scheduled["definition_json"]["default_status"] == "paused" + assert scheduled["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + assert by_name["edge_platform_orders"]["status"] == "paused" + assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" + assert by_name["edge_platform_orders"]["definition_json"]["when"] == { + "kind": "platform_event", + "kinds": ["tick_overdue"], + } @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index a95849fa86..edae5b6127 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -64,6 +64,27 @@ expected_child_statuses={"staging": ("success",), "marts": ("success",)}, expected_totals=("success", None), ), + GroupExecutionTestCase( + "a platform sensor starts group parents and children", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_origin="sensor", + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, job_group, platform_event, sensor\n" + "staging = job('staging', description='Stage orders.', select='stg_orders broken_orders')\n" + "marts = job('marts', description='Build independent marts.', select='orders totals')\n" + "annual_orders = job_group('annual_orders', description='Stage then build marts.', jobs=[marts, staging])\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), GroupExecutionTestCase( "whole downstream job is blocked without building an empty selection", "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", @@ -161,6 +182,9 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail runs["marts"]["started_at"] or runs["staging"]["ended_at"] ) with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (parent["run_id"],) + ).fetchone() == (test_case.expected_origin,) assert ( control.execute( "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" @@ -379,6 +403,23 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "success", [(23,)], ), + WorkerExecutionTestCase( + "a platform sensor catches up retained events through a real worker", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.')\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), WorkerExecutionTestCase( "authored variables and reload use ordinary build argument conversion", "SELECT @@batch AS amount", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index f274c11f7e..68b97e7b34 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -603,6 +603,34 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "edge_inventory_group": {"edge_snapshot_inventory": []}, "edge_products_group": {"edge_snapshot_products": []}, }, + expected_sensors={ + "edge_platform_orders": { + "description": "Catch up retained platform observations with a stopped group sensor.", + "job_key": "edge_snapshot_group", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 7, + "default_status": "stopped", + } + }, + expected_schedules=("edge_snapshot_orders",), + ), + SnapshotDiscoveryTestCase( + description="sensor-only roots preserve referenced jobs and filter targets", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text() + }, + expected_jobs=("health_orders",), + expected_sensors={ + "overdue_orders": { + "description": "Catch up overdue order reporting.", + "job_key": "health_orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 7, + "default_status": "stopped", + } + }, ), SnapshotDiscoveryTestCase( description="a project without declarations has no job document", @@ -634,6 +662,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects name: definition["jobs"] for name, definition in publication["documents"].get("groups", {}).items() } == test_case.expected_groups + assert { + name: publication["documents"]["triggers"][name] for name in test_case.expected_sensors + } == test_case.expected_sensors + assert tuple(publication["documents"].get("triggers", {})) == tuple( + sorted((*test_case.expected_schedules, *test_case.expected_sensors)) + ) for definition in publication["documents"].get("jobs", {}).values(): assert set(definition["selection"]) <= publication["nodes"].keys() @@ -720,6 +754,54 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="mutually exclusive", ), + SnapshotDiscoveryRejectionTestCase( + description="a zero sensor interval cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("minimum_interval_seconds=7", "minimum_interval_seconds=0") + }, + expected_error=ValueError, + expected_message="nonzero u32", + ), + SnapshotDiscoveryRejectionTestCase( + description="empty platform conditions cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") + }, + expected_error=ValueError, + expected_message="nonempty event kinds", + ), + SnapshotDiscoveryRejectionTestCase( + description="platform kinds are a closed vocabulary", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + SnapshotDiscoveryRejectionTestCase( + description="sensor code defaults use running or stopped", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace('default_status="stopped"', 'default_status="paused"') + }, + expected_error=ValueError, + expected_message="unknown variant", + ), ], ids=lambda case: case.description, ) From 39eebcecfb1bb3a84ccb9b7ed948995ded1c25d6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:11:11 +0000 Subject: [PATCH 049/138] feat: observe expired code location heartbeats --- .../orchestration/edge_cases/definitions.py | 2 +- .../src/platform/_helpers/liveness.rs | 59 ++++++ .../src/platform/_helpers/mod.rs | 1 + .../src/platform/_helpers/watchdog.rs | 11 +- .../src/platform/models.rs | 16 ++ .../tests/platform/test_types.rs | 6 + .../tests/platform/test_watchdog.rs | 176 +++++++++++++++++- .../tests/sensors/helpers.rs | 12 +- .../orchestration_declarations/types.py | 1 + .../publication/test_publication.py | 2 +- .../test_orchestration_snapshot.py | 5 +- 11 files changed, 283 insertions(+), 8 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index ef8caed097..f88b56bb70 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -61,7 +61,7 @@ edge_health = sensor( "edge_platform_orders", description="Catch up retained platform observations with a stopped group sensor.", - when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE, PlatformEventKind.LOCATION_BROKEN]), run=edge_group, minimum_interval_seconds=7, default_status="stopped", diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs new file mode 100644 index 0000000000..7239f2d718 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs @@ -0,0 +1,59 @@ +//! Liveness is derived across all sessions; an old instance cannot mask a healthy successor. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{PlatformEvent, WatchdogLocation}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Locations report every 30 seconds; tolerate two missed reports. +pub(crate) async fn scan_location_page( + store: &ControlStore, + installation_id: Uuid, + after: Option<(Uuid, String)>, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let mut parameters = vec![Parameter::Time(transaction.now - TimeDelta::seconds(90))]; + let cursor = if let Some((location_id, target)) = after { + parameters.extend([Parameter::Id(location_id), Parameter::Text(target)]); + "AND (i.location_id>$2 OR (i.location_id=$2 AND i.target>$3))" + } else { + "" + }; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT i.location_id, i.target, MAX(i.last_seen_at) AS last_seen_at \ + FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE l.archived_at IS NULL {cursor} GROUP BY i.location_id,i.target \ + HAVING MAX(i.last_seen_at)<$1 ORDER BY i.location_id,i.target LIMIT 100" + ), + parameters, + ) + .await?; + let locations: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = locations + .last() + .map(|row| (row.location_id, row.target.clone())); + for location in locations { + transaction = observe( + transaction, + installation_id, + PlatformEvent::LocationBroken { + location_id: location.location_id, + target: location.target, + last_seen_at: location.last_seen_at, + }, + ) + .await?; + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index 06cdb8c059..574aaa4680 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod liveness; pub(crate) mod location_heartbeat; pub(crate) mod location_work; pub(crate) mod observations; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index 16ded206d3..89ca089b53 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -1,10 +1,11 @@ -//! Health scanning releases the writer lock between bounded trigger pages. +//! Health scanning releases the writer lock between location and trigger pages. use chrono::TimeDelta; use tokio::sync::watch; use uuid::Uuid; use crate::declarations::models::Schedule; +use crate::platform::_helpers::liveness::scan_location_page; use crate::platform::_helpers::observations::observe; use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; use crate::store::errors::ControlError; @@ -40,6 +41,14 @@ pub(crate) async fn scan_health( .await?; } commit(transaction).await?; + let mut location_after = None; + while !*shutdown.borrow() { + location_after = scan_location_page(store, health.installation_id, location_after).await?; + if location_after.is_none() { + break; + } + tokio::task::yield_now().await; + } let mut after = None; while !*shutdown.borrow() { after = scan_trigger_page(store, health.installation_id, after).await?; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 09ded290f5..39f6851841 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -119,6 +119,7 @@ pub enum PlatformEventKind { SchedulerNoLeader, TickOverdue, CalendarHorizonLow, + LocationBroken, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -137,6 +138,12 @@ pub enum PlatformEvent { refresh_at: DateTime, horizon_end: Option>, }, + /// All observed instances of this location/target have missed three heartbeats. + LocationBroken { + location_id: Uuid, + target: String, + last_seen_at: DateTime, + }, } impl PlatformEvent { @@ -147,6 +154,7 @@ impl PlatformEvent { Self::SchedulerNoLeader => PlatformEventKind::SchedulerNoLeader, Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, + Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, } } @@ -157,6 +165,7 @@ impl PlatformEvent { Self::SchedulerNoLeader => "platform_scheduler_no_leader", Self::TickOverdue { .. } => "platform_tick_overdue", Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", + Self::LocationBroken { .. } => "platform_location_broken", } } } @@ -174,6 +183,13 @@ pub(crate) struct WatchdogTrigger { pub(crate) next_due_at: Option>, } +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogLocation { + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) last_seen_at: DateTime, +} + /// The endpoint may contain credentials; neither errors nor Debug reveal it. #[derive(Clone)] pub struct PlatformWebhook { diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 970ec7a8c0..52ecb92e0c 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -15,6 +15,12 @@ pub(crate) struct WatchdogTestCase { pub(crate) expected_low_horizons: usize, } +pub(crate) struct LocationWatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_broken: usize, +} + #[derive(Debug, Deserialize)] pub(crate) struct Observation { pub(crate) storage_id: i64, diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 0d03b0e495..fa3574df89 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -1,16 +1,22 @@ use std::collections::BTreeSet; use std::time::Duration; +use crate::platform::main::report_location_health::report_location_health; use crate::platform::main::run_watchdog::run_watchdog; -use crate::platform::models::PlatformEvent; +use crate::platform::models::{LocationHealth, LocationHealthReport, PlatformEvent}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use chrono::TimeDelta; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::platform::tests::helpers::{snapshot, wait_for_scans}; -use crate::platform::tests::test_types::WatchdogTestCase; +use crate::platform::tests::test_types::{LocationWatchdogTestCase, WatchdogTestCase}; use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -169,3 +175,169 @@ fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_du }); } } + +#[test] +fn given_stale_location_sessions_when_watchdogs_scan_then_latest_target_liveness_controls_paged_outage_events() + { + let test_cases = [ + LocationWatchdogTestCase { + description: "SQLite latest target receipts", + backend: Backend::Sqlite, + expected_broken: 104, + }, + LocationWatchdogTestCase { + description: "PostgreSQL latest target receipts", + backend: Backend::Postgres, + expected_broken: 104, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let archived = create_location(store, "inventory", "test-admin") + .await + .expect("archived location"); + create_location(store, "products", "test-admin") + .await + .expect("never observed location"); + for (location_id, count) in [(location, 105), (archived, 1)] { + for index in 0..count { + for _ in 0..2 { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: format!("orders_{index:03}"), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register old session"); + } + } + } + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age sessions"); + let stale_at = transaction.now - TimeDelta::seconds(105); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1", + vec![Parameter::Time(stale_at)], + ) + .await + .expect("age all old sessions"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_locations SET archived_at=$1 WHERE location_id=$2", + vec![Parameter::Time(now), Parameter::Id(archived)], + ) + .await + .expect("archive location"); + commit(transaction).await.expect("commit fixture"); + // A fresh successor protects only its target, not another target of the same location. + let healthy_session = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: healthy_session, + target: "orders_104".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("healthy successor"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age successor"); + let recent_at = transaction.now - TimeDelta::seconds(75); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1 WHERE instance_id=$2", + vec![Parameter::Time(recent_at), Parameter::Id(healthy_session)], + ) + .await + .expect("two missed heartbeats"); + commit(transaction).await.expect("commit successor time"); + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { run_watchdog(&store, id, shutdown).await }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let broken: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::LocationBroken { + location_id, + ref target, + last_seen_at, + } = row.payload.platform + else { + return None; + }; + assert_eq!( + location_id, location, + "archived and never-observed locations are excluded" + ); + assert_eq!( + last_seen_at, stale_at, + "payload retains the latest server receipt time" + ); + Some(target.clone()) + }) + .collect(); + assert_eq!( + broken, + (0..test_case.expected_broken) + .map(|index| format!("orders_{index:03}")) + .collect(), + "{}: both pages are read, old replicas deduplicate, and the healthy target is excluded", + test_case.description + ); + let keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("identity"), + ) + }) + .collect(); + assert_eq!( + keys.len(), + observations.len(), + "replicas share observation keys" + ); + stop.send(true).expect("stop replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join") + .expect("watchdog stopped"); + } + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 7aea4d6287..5152689454 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -29,7 +29,7 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { json!({ "health_orders": { "description": "Refresh reporting on overdue ticks", "job_key": "orders", - "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken"]}, "minimum_interval_seconds": 1 } }), @@ -53,9 +53,17 @@ pub(crate) async fn append_events( PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(8), }, + PlatformEvent::LocationBroken { + location_id: Uuid::from_u128(9), + target: "development".into(), + last_seen_at: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(10), + }, ]; for index in indices { - let platform = &payloads[(index % 2) as usize]; + let platform = &payloads[(index % 4) as usize]; let (next, appended) = append_control_event( transaction, ControlEvent { diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index c41199fb86..edba630aa1 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -10,3 +10,4 @@ class PlatformEventKind(StrEnum): SCHEDULER_NO_LEADER = "scheduler_no_leader" TICK_OVERDUE = "tick_overdue" CALENDAR_HORIZON_LOW = "calendar_horizon_low" + LOCATION_BROKEN = "location_broken" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 0c49401511..1f33e90df2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -133,7 +133,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" assert by_name["edge_platform_orders"]["definition_json"]["when"] == { "kind": "platform_event", - "kinds": ["tick_overdue"], + "kinds": ["tick_overdue", "location_broken"], } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 68b97e7b34..72f6ded1d2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -607,7 +607,10 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "edge_platform_orders": { "description": "Catch up retained platform observations with a stopped group sensor.", "job_key": "edge_snapshot_group", - "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "when": { + "kind": "platform_event", + "kinds": ["tick_overdue", "location_broken"], + }, "minimum_interval_seconds": 7, "default_status": "stopped", } From 80cbc0c2dff6e66f3b4b039dd5a11ffb75e33623 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:41:09 +0000 Subject: [PATCH 050/138] feat: retain typed launch failure observations Publish scheduler launch observations atomically with their source facts and retain pending observations across publication failures. Allow late observations only with immutable proof of the exact recorded claim and current scheduler fencing. Terminal projections and physical worker ownership remain unchanged. Cover local launch failures, real Docker response-loss and worker-completion ordering, transactional retry faults, and SQLite/PostgreSQL authority boundaries. --- .../orchestration/edge_cases/definitions.py | 8 +- .../src/workers/_helpers/recovery.rs | 8 + .../src/workers/_helpers/shutdown.rs | 18 ++ .../src/workers/_helpers/supervisor.rs | 50 ++++- crates/sqlbuild-cli/src/workers/models.rs | 1 + .../native_commands/worker/test_recovery.rs | 41 +++- .../native_commands/worker/test_types.rs | 2 + .../src/platform/models.rs | 9 + .../src/runs/_helpers/lifecycle/changes.rs | 34 +++ .../src/runs/_helpers/lifecycle/projection.rs | 1 + .../runs/_helpers/lifecycle/publish_change.rs | 37 ++- .../src/runs/main/change_run.rs | 10 +- .../sqlbuild-orchestration/src/runs/models.rs | 5 + .../tests/runs/test_lifecycle.rs | 210 ++++++++++++++++++ .../tests/sensors/helpers.rs | 12 +- .../orchestration_declarations/types.py | 3 +- .../publication/docker/helpers.py | 66 ++++++ .../publication/test_docker_worker.py | 81 +++++++ .../publication/test_publication.py | 2 +- .../test_orchestration_snapshot.py | 2 +- 20 files changed, 580 insertions(+), 20 deletions(-) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index f88b56bb70..6762839dd4 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -61,7 +61,13 @@ edge_health = sensor( "edge_platform_orders", description="Catch up retained platform observations with a stopped group sensor.", - when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE, PlatformEventKind.LOCATION_BROKEN]), + when=platform_event( + kinds=[ + PlatformEventKind.TICK_OVERDUE, + PlatformEventKind.LOCATION_BROKEN, + PlatformEventKind.LAUNCH_FAILED, + ] + ), run=edge_group, minimum_interval_seconds=7, default_status="stopped", diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs index c54d87b613..f439a58dd6 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -63,12 +63,20 @@ impl Workers { "Worker supervision recovered for run {}", claim.request.run_id )); + let rejected = child.is_none(); + let index = self.active.len(); self.active.push(ClaimedWorker { claim, child, stopping: None, exit: None, + launch_failure: rejected.then_some("docker_configuration_rejected"), }); + if rejected { + self.active[index] + .publish_launch_failure(store, lease) + .await?; + } } } } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs index 9613633420..7eafd6f876 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -114,6 +114,24 @@ impl Workers { } renewed = Instant::now(); } + if self.active[index].launch_failure.is_some() { + let reported = match lease { + Some(lease) => { + self.active[index] + .publish_launch_failure(store, lease) + .await + } + None => Err(ControlError::Fenced), + }; + if let Err(error) = reported { + if matches!(error, ControlError::Fenced) { + failure = Some(error); + } else { + failure.get_or_insert(error); + } + continue; + } + } if let Some(WorkerChild::Docker(child)) = &self.active[index].child { match child.try_wait().await { Ok(None) => { diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index b9aa97e091..ab053c0d10 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -177,10 +177,14 @@ impl Workers { child: None, stopping: None, exit: None, + launch_failure: None, }); let worker = &mut self.active[index]; let claim = &worker.claim; if matches!(location, Launcher::Local(_)) && claim.request.image_id.is_some() { + worker.launch_failure = Some("local_image_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; self.active.swap_remove(index); continue; @@ -193,6 +197,9 @@ impl Workers { Some(name) } Err(_) => { + worker.launch_failure = Some("docker_configuration_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; publish(store, lease, claim, ("docker-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "Docker worker requires a pinned image and supported launcher configuration".into(), exit_code: None, })).await?; @@ -220,12 +227,19 @@ impl Workers { ) .await?; let local_command = match location { - Launcher::Local(location) => Some(prepare_spawn( + Launcher::Local(location) => match prepare_spawn( location, claim, &credential.bearer, &self.control_variable, - )?), + ) { + Ok(command) => Some(command), + Err(error) => { + worker.launch_failure = Some("local_preparation_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); + } + }, Launcher::Docker(_) => None, }; publish( @@ -245,10 +259,13 @@ impl Workers { None => { if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = (location, &worker.child) - { - child + && let Err(error) = child .launch(location, claim, &credential.bearer, &self.control_variable) - .await?; + .await + { + worker.launch_failure = Some("docker_launch_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); } Ok(()) } @@ -262,6 +279,9 @@ impl Workers { )); } Err(_) => { + worker.launch_failure = Some("local_spawn_failed"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; publish( store, lease, @@ -311,6 +331,26 @@ fn prepare_spawn( Ok(command) } +impl ClaimedWorker { + pub(in crate::workers) async fn publish_launch_failure( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if let Some(code) = self.launch_failure { + publish( + store, + lease, + &self.claim, + (code, RunChange::LaunchFailed { code: code.into() }), + ) + .await?; + self.launch_failure = None; + } + Ok(()) + } +} + pub(in crate::workers) async fn publish( store: &ControlStore, lease: &SchedulerLease, diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 415247a7f7..35639fe18e 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -98,6 +98,7 @@ pub(super) struct ClaimedWorker { pub(super) child: Option, pub(super) stopping: Option, pub(super) exit: Option, + pub(super) launch_failure: Option<&'static str>, } /// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs index 0cdf1611ca..0a723a1db4 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -14,11 +14,29 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ WorkerRecoveryTestCase { description: "snapshot without an image pin", image_id: None, + failure_event: "run_terminated", + expected_pending_observations: 1, expected_status: "failed", }, WorkerRecoveryTestCase { description: "snapshot with a malformed image pin", image_id: Some("sha256:invalid"), + failure_event: "run_terminated", + expected_pending_observations: 1, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "missing image pin retains a rolled-back launch observation", + image_id: None, + failure_event: "platform_launch_failed", + expected_pending_observations: 0, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "invalid image pin retains a rolled-back launch observation", + image_id: Some("sha256:invalid"), + failure_event: "platform_launch_failed", + expected_pending_observations: 0, expected_status: "failed", }, ]; @@ -37,11 +55,11 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ connection .busy_timeout(Duration::from_secs(5)) .expect("database contention timeout"); - connection.execute_batch( + connection.execute_batch(&format!( "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00Z'; CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events - WHEN NEW.event_type='run_terminated' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", - ).expect("expire the departed leader and interrupt terminal publication"); + WHEN NEW.event_type='{}' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", test_case.failure_event + )).expect("expire the departed leader and interrupt durable publication"); let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; let held: (i64, i64) = connection.query_row( @@ -49,6 +67,10 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ [], |row| Ok((row.get(0)?, row.get(1)?)), ).expect("retained claim while publication is unavailable"); assert_eq!(held, (1, 1), "{}", test_case.description); + for event_type in ["run_launch_failed", "platform_launch_failed"] { + let pending: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type=?1", [event_type], |row| row.get(0)).expect("pending source/platform pair"); + assert_eq!(pending, test_case.expected_pending_observations, "{}: {event_type}", test_case.description); + } connection .execute_batch("DROP TRIGGER reject_recovery_publication") .expect("restore terminal publication"); @@ -69,6 +91,19 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ "{}", test_case.description ); + let (producer, payload, count): (String, String, i64) = connection.query_row( + "SELECT producer, payload, COUNT(*) OVER () FROM recall_events WHERE event_type='platform_launch_failed'", + [], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ).expect("durable launch observation from the real replacement scheduler"); + assert_eq!(producer, "platform"); + assert_eq!(count, 1, "{}", test_case.description); + let source_count: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type='run_launch_failed'", [], |row| row.get(0)).expect("retried source fact"); + assert_eq!(source_count, 1, "{}", test_case.description); + let event: serde_json::Value = serde_json::from_str(&payload).expect("typed event payload"); + assert_eq!(event["platform"]["kind"], "launch_failed"); + assert_eq!(event["platform"]["run_id"], worker.identity.0.to_string()); + assert_eq!(event["platform"]["claim_id"], worker.identity.1.to_string()); + assert_eq!(event["platform"]["code"], "docker_configuration_rejected"); assert!( std::fs::read_to_string(&replacement.stderr) .expect("shutdown recovery log") diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs index 81ecbc260f..0a6741cff7 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -16,6 +16,8 @@ pub(crate) struct WorkerHeartbeatTestCase { pub(crate) struct WorkerRecoveryTestCase { pub(crate) description: &'static str, pub(crate) image_id: Option<&'static str>, + pub(crate) failure_event: &'static str, + pub(crate) expected_pending_observations: i64, pub(crate) expected_status: &'static str, } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 39f6851841..7ab2762ac9 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -120,6 +120,7 @@ pub enum PlatformEventKind { TickOverdue, CalendarHorizonLow, LocationBroken, + LaunchFailed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -144,6 +145,12 @@ pub enum PlatformEvent { target: String, last_seen_at: DateTime, }, + /// A launch attempt or its response failed; the worker may still start or finish. + LaunchFailed { + run_id: Uuid, + claim_id: Uuid, + code: String, + }, } impl PlatformEvent { @@ -155,6 +162,7 @@ impl PlatformEvent { Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, + Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed, } } @@ -166,6 +174,7 @@ impl PlatformEvent { Self::TickOverdue { .. } => "platform_tick_overdue", Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", Self::LocationBroken { .. } => "platform_location_broken", + Self::LaunchFailed { .. } => "platform_launch_failed", } } } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index 8310532774..f6a099cf26 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -4,8 +4,41 @@ use crate::runs::models::{ ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; +/// A launch-response observation is append-only even after completion, but requires recorded ownership. +pub(crate) async fn check_launch_claim( + transaction: ControlTransaction, + request: &RunChangeRequest, +) -> Result { + let claim_id = request.claim_id.ok_or_else(|| { + ControlError::Invalid("launch observation requires a recorded claim".into()) + })?; + let (transaction, rows) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 AND run_id=$2 \ + AND event_type='run_claimed' AND producer='scheduler' AND invocation_id=$3", + vec![ + Parameter::Text(claim_id.to_string()), + Parameter::Text(request.run_id.to_string()), + Parameter::Text(format!("scheduler/{}", request.run_id)), + ], + ) + .await?; + let recorded = rows.first().map(|row| &row["payload"]["claim"]); + if recorded.is_none_or(|claim| { + claim["claim_id"] != claim_id.to_string() + || claim["request"]["run_id"] != request.run_id.to_string() + }) { + return Err(ControlError::Invalid( + "launch observation does not belong to a recorded claim".into(), + )); + } + Ok(transaction) +} + pub(crate) fn validate_retry( previous: &Row, request: &RunChangeRequest, @@ -85,6 +118,7 @@ pub(crate) fn validate_change( )); } let valid = match request.change { + RunChange::LaunchFailed { .. } => false, RunChange::Launched { .. } => state.worker_instance.is_some(), RunChange::Started => { state.worker_instance.is_some() && matches!(state.status, Queued | Launched) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index 7f6ffb2de0..b430f35721 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -17,6 +17,7 @@ pub(crate) async fn project_change( ) -> Result { let now = transaction.now; match &request.change { + RunChange::LaunchFailed { .. } => {} RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { return project_node(transaction, request, &state.node_counts_json).await; } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 68cc3edd32..c5445432e7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -1,26 +1,57 @@ -//! Publish a lifecycle fact, its projection and terminal cleanup in the caller's transaction. +//! Publish facts atomically; launch observations never project state or release physical ownership. use serde_json::json; +use crate::platform::models::PlatformEvent; use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::_helpers::lifecycle::projection::project_change; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; pub(crate) async fn publish_change( transaction: ControlTransaction, request: &RunChangeRequest, state: &ChangingRun, ) -> Result { - let (transaction, mut cursor) = append_run_event( + let (mut transaction, mut cursor) = append_run_event( transaction, (request.run_id, request.event_id), request.change.event_type(), json!({"change": request}), ) .await?; + if let (RunChange::LaunchFailed { code }, Some(claim_id)) = (&request.change, request.claim_id) + { + let platform = PlatformEvent::LaunchFailed { + run_id: request.run_id, + claim_id, + code: code.clone(), + }; + let occurred_at = transaction.now; + let outcome; + (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id: uuid::Uuid::new_v5(&request.event_id, b"platform-launch-failed"), + invocation_id: format!("platform/{}", request.run_id), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: Some(request.run_id), + payload: json!({"platform": platform}), + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "launch failure conflicts with a durable fact".into(), + )); + } + return Ok(transaction); + } let mut transaction = project_change(transaction, request, state).await?; let mut cancel_unstarted = matches!(request.change, RunChange::CancelRequested { .. }) && state.worker_instance.is_none(); diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index c43086efe7..59cb4f2a40 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -4,7 +4,7 @@ use serde_json::json; use crate::credentials::main::append_audit::append_audit; use crate::runs::_helpers::lifecycle::changes::{ - validate_authority, validate_change, validate_retry, + check_launch_claim, validate_authority, validate_change, validate_retry, }; use crate::runs::_helpers::lifecycle::publish_change::publish_change; use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; @@ -39,7 +39,7 @@ pub async fn change_run( commit(transaction).await?; return Ok(false); } - let (transaction, rows) = query( + let (mut transaction, rows) = query( transaction, "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", @@ -51,7 +51,11 @@ pub async fn change_run( .next() .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, )?; - validate_change(&state, request)?; + if matches!(request.change, RunChange::LaunchFailed { .. }) { + transaction = check_launch_claim(transaction, request).await?; + } else { + validate_change(&state, request)?; + } let mut transaction = publish_change(transaction, request, &state).await?; if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = (authority, &request.change) diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 64ab1a370c..16625f5cc1 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -250,6 +250,10 @@ pub enum RunChange { Launched { container_id: Option, }, + /// A failed launch response is not physical evidence that the worker has stopped. + LaunchFailed { + code: String, + }, Started, NodeStarted { node_key: String, @@ -279,6 +283,7 @@ impl RunChange { pub(crate) const fn event_type(&self) -> &'static str { match self { Self::Launched { .. } => "run_launched", + Self::LaunchFailed { .. } => "run_launch_failed", Self::Started => "run_started", Self::NodeStarted { .. } => "run_node_started", Self::NodeFinished { .. } => "run_node_finished", diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 2f98d185eb..21ec54e845 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use crate::credentials::models::AuditContext; +use crate::platform::models::PlatformEvent; use crate::runs::main::change_run::change_run; use crate::runs::main::claim_next_run::claim_next_run; use crate::runs::main::heartbeat_run::heartbeat_run; @@ -11,6 +12,7 @@ use crate::runs::models::{ }; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; @@ -19,6 +21,7 @@ use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::helpers::{queue, request, snapshot}; use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -118,6 +121,96 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .status, RunStatus::Running ); + let launch_failed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::LaunchFailed { + code: "docker_launch_failed".into(), + }, + ..started.clone() + }; + assert!(change_run(store, worker, &launch_failed).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &launch_failed) + .await + .is_err() + ); + let wrong_claim = RunChangeRequest { + claim_id: Some(Uuid::now_v7()), + ..launch_failed.clone() + }; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &wrong_claim) + .await + .is_err() + ); + let before_launch_failure = read_recall_events(store, 0, 100) + .await + .expect("before launch failure"); + let fault = TransactionFault { + description: "platform launch observation must commit with its run fact", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='platform_launch_failed'", + sqlite_body: "SELECT RAISE(ABORT,'injected launch observation failure')", + postgres_body: "RAISE EXCEPTION 'injected launch observation failure'", + }; + install_fault(store, &fault).await; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back launch fact"), + before_launch_failure + ); + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("launch observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("idempotent retry") + ); + let observations: Vec<_> = read_recall_events(store, 0, 100) + .await + .expect("health facts") + .into_iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .collect(); + assert_eq!(observations.len(), 1); + let platform: PlatformEvent = + serde_json::from_value(observations[0]["payload"]["platform"].clone()) + .expect("typed launch failure"); + assert_eq!( + platform, + PlatformEvent::LaunchFailed { + run_id: started.run_id, + claim_id: claim.claim_id, + code: "docker_launch_failed".into() + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("live run") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run") + .status, + RunStatus::Running + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("physical ownership retained") + .is_none() + ); let success = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::Finished { @@ -345,6 +438,123 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ assert_eq!(nodes[0].status, "success"); assert_eq!(nodes[0].rows_affected, Some(17)); assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let late_failure = RunChangeRequest { + event_id: Uuid::now_v7(), + ..launch_failed.clone() + }; + let before_late = read_recall_events(store, 0, 100) + .await + .expect("terminal facts"); + let expiry = TransactionFault { + description: "late launch observation loses leadership before commit", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ..fault + }; + for (point, fenced) in [(&fault, false), (&expiry, true)] { + install_fault(store, point).await; + let failed = + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await; + let error = failed.expect_err(point.description); + assert_eq!( + matches!(error, ControlError::Fenced), + fenced, + "{}", + point.description + ); + remove_fault(store, point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("terminal rollback"), + before_late + ); + } + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late retry") + ); + assert!(change_run(store, worker, &late_failure).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &late_failure) + .await + .is_err() + ); + for invalid in [ + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: None, + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + claim_id: Some(Uuid::new_v5(&Uuid::from_u128(2), b"run_claimed")), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..started.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..launched.clone() + }, + ] { + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &invalid) + .await + .is_err(), + "terminal authority remains narrow: {invalid:?}" + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("unchanged terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"), + projected + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("late fact pair"); + assert_eq!( + recorded + .iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .count(), + 2 + ); + expire_lease(store).await; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await, + Err(ControlError::Fenced) + ), + "even duplicate observations require a live leader" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect successor") + .expect("successor"); let following = claim_next_run(store, &lease, &limits, None) .await .expect("claim following run") diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 5152689454..3f304e66c7 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -29,7 +29,7 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { json!({ "health_orders": { "description": "Refresh reporting on overdue ticks", "job_key": "orders", - "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken"]}, + "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"]}, "minimum_interval_seconds": 1 } }), @@ -61,9 +61,17 @@ pub(crate) async fn append_events( PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(10), }, + PlatformEvent::LaunchFailed { + run_id: Uuid::from_u128(11), + claim_id: Uuid::from_u128(12), + code: "local_spawn_failed".into(), + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(13), + }, ]; for index in indices { - let platform = &payloads[(index % 4) as usize]; + let platform = &payloads[(index % 6) as usize]; let (next, appended) = append_control_event( transaction, ControlEvent { diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index edba630aa1..913d56da06 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -4,10 +4,11 @@ class PlatformEventKind(StrEnum): - """Closed vocabulary of durable platform observations.""" + """Launch failures include response errors and do not prove worker absence.""" HEARTBEAT = "heartbeat" SCHEDULER_NO_LEADER = "scheduler_no_leader" TICK_OVERDUE = "tick_overdue" CALENDAR_HORIZON_LOW = "calendar_horizon_low" LOCATION_BROKEN = "location_broken" + LAUNCH_FAILED = "launch_failed" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py index d1ac635fdd..36d2ea7541 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -3,11 +3,16 @@ import json import os import shutil +import socket import subprocess import sys from collections.abc import Iterator from contextlib import contextmanager +from http.client import HTTPConnection, HTTPResponse +from http.server import BaseHTTPRequestHandler from pathlib import Path +from socketserver import UnixStreamServer +from threading import Event, Thread from uuid import uuid4 import pytest @@ -126,3 +131,64 @@ def worker_container(location: DockerLocation, location_id: str, run_id: str) -> == location.image_id ) return result + + +@contextmanager +def lost_start_response(directory: Path, upstream: Path, inspection: Event) -> Iterator[Path]: + """Fail the real start result and hold physical inspection until explicitly restored.""" + proxy: Path = directory / "docker-proxy.sock" + + class Handler(BaseHTTPRequestHandler): + def forward(self) -> None: + connection: HTTPConnection = HTTPConnection("localhost", timeout=5) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(5) + connection.sock.connect(str(upstream)) + try: + body: bytes = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + connection.request( + self.command, self.path, body, {"Content-Type": "application/json"} + ) + response: HTTPResponse = connection.getresponse() + payload: bytes = response.read() + status: int = response.status + started: bool = self.command == "POST" and self.path.endswith("/start") + unavailable: bool = ( + self.command == "GET" + and self.path.startswith("/containers/sqlbuild-") + and not inspection.is_set() + ) + assert not started or status == 204, ( + "The actual Docker start must succeed before faulting" + ) + status, payload = { + (False, False): (status, payload), + (True, False): (503, b'{"message":"synthetic lost start response"}'), + (False, True): (503, b'{"message":"synthetic inspection outage"}'), + }[(started, unavailable)] + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + finally: + connection.close() + + def do_GET(self) -> None: + self.forward() + + def do_POST(self) -> None: + self.forward() + + def log_message(self, format: str, *args: object) -> None: + """Never copy credential-bearing Docker requests to test output.""" + + with UnixStreamServer(str(proxy), Handler) as server: + thread: Thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield proxy + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Docker fault proxy did not stop" + proxy.unlink(missing_ok=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index 8d54cf22ee..05a6f35e07 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -4,19 +4,26 @@ import sqlite3 import sys import time +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path +from threading import Event from typing import Any import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.publication import helpers from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ScheduledWorker, + WorkerDeployment, WorkerExecutionTestCase, WorkerRestartTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( REQUIRES_DOCKER, docker_command, + docker_deployment, + lost_start_response, worker_container, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( @@ -250,3 +257,77 @@ def test_given_an_executing_container_when_scheduler_restarts_then_the_same_work assert control.execute( "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" ).fetchone() == (1,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "worker completes before the lost launch response can be durably observed", + "SELECT 37 AS amount", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + store_setup=( + "CREATE TRIGGER reject_launch_observation BEFORE INSERT ON recall_events " + "WHEN NEW.event_type='platform_launch_failed' " + "BEGIN SELECT RAISE(ABORT,'synthetic observation outage'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_lost_docker_start_response_when_the_worker_finishes_then_late_observation_retries_do_not_revive_it( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + inspection: Event = Event() + + @contextmanager + def deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + with docker_deployment(directory, executable) as actual: + assert actual.docker is not None + with lost_start_response(directory, actual.docker.socket, inspection) as proxy: + yield actual._replace(configuration=actual.configuration | {"socket": str(proxy)}) + + monkeypatch.setattr(helpers, "docker_deployment", deployment) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + before: list[tuple[Any, ...]] = control.execute("SELECT * FROM sqb_runs").fetchall() + assert control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type IN ('run_launch_failed','platform_launch_failed')" + ).fetchone() == (0,) + _ = control.execute("DROP TRIGGER reject_launch_observation") + deadline: float = time.monotonic() + 10 + while control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type='platform_launch_failed'" + ).fetchone() != (1,): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + # Terminal state alone cannot bypass the still-unavailable physical observation. + time.sleep(1.2) + assert worker.process.poll() is None, worker.log.read_text() + assert control.execute("SELECT * FROM sqb_runs").fetchall() == before + assert control.execute( + "SELECT event_type FROM recall_events WHERE event_type IN " + "('run_terminated','run_launch_failed','platform_launch_failed') ORDER BY storage_id" + ).fetchall() == [ + ("run_terminated",), + ("run_launch_failed",), + ("platform_launch_failed",), + ] + assert control.execute("SELECT COUNT(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT COUNT(*) FROM sqb_run_liveness").fetchone() == (0,) + inspection.set() + assert worker.process.wait(timeout=5) != 0, "Original response failure remains visible" + assert worker.docker is not None + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + assert state["Running"] is False + assert state["ExitCode"] == 0 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 1f33e90df2..0812a7a2f5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -133,7 +133,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" assert by_name["edge_platform_orders"]["definition_json"]["when"] == { "kind": "platform_event", - "kinds": ["tick_overdue", "location_broken"], + "kinds": ["tick_overdue", "location_broken", "launch_failed"], } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 72f6ded1d2..48ab859069 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -609,7 +609,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "job_key": "edge_snapshot_group", "when": { "kind": "platform_event", - "kinds": ["tick_overdue", "location_broken"], + "kinds": ["tick_overdue", "location_broken", "launch_failed"], }, "minimum_interval_seconds": 7, "default_status": "stopped", From 91a5eb35ecb254f492043da3472ab423ff25993d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:09:36 +0000 Subject: [PATCH 051/138] feat: add scoped job and group outcome sensors --- .../orchestration/edge_cases/definitions.py | 18 + .../src/sensors/_helpers/mod.rs | 1 + .../src/sensors/_helpers/platform.rs | 5 +- .../src/sensors/_helpers/runs.rs | 49 +++ .../sensors/main/complete_builtin_sensor.rs | 12 +- .../src/sensors/main/validate_sensors.rs | 11 +- .../src/sensors/models.rs | 46 ++- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 4 +- .../src/store/run_sensors.sql | 2 + .../tests/api/test_location_work.rs | 4 +- .../tests/sensors/helpers.rs | 109 ++++++ .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_runs.rs | 324 ++++++++++++++++++ .../tests/snapshots/helpers.rs | 3 +- .../_helpers/native_host/constants.rs | 2 + .../native_host/orchestration_declarations.rs | 17 +- .../native_host/orchestration_triggers.rs | 6 +- python-boundary.toml | 2 +- src/sqlbuild/orchestration.py | 7 +- .../orchestration_declarations/models.py | 13 +- .../orchestration_declarations/types.py | 12 + .../compiler/compiled_project/_test_types.py | 4 +- .../compiled_project/publication/helpers.py | 16 +- .../publication/test_publication.py | 21 +- .../publication/test_worker.py | 54 +++ .../test_orchestration_snapshot.py | 69 +++- 27 files changed, 785 insertions(+), 29 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs create mode 100644 crates/sqlbuild-orchestration/src/store/run_sensors.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_runs.rs diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 6762839dd4..8895ffef72 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -2,9 +2,11 @@ from sqlbuild.orchestration import ( PlatformEventKind, + RunEventKind, job, job_group, platform_event, + run_event, schedule, sensor, ) @@ -73,3 +75,19 @@ default_status="stopped", ) edge_factory_results.append(edge_health) + +edge_job_outcomes = sensor( + "edge_job_outcomes", + description="Keep warning and partial outcomes distinct from success.", + when=run_event(job=edge_orders, kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), + run=edge_inventory, + default_status="stopped", +) +edge_group_outcomes = sensor( + "edge_group_outcomes", + description="Observe group completion including cancellation and presumed failure.", + when=run_event(job=edge_group, kinds=[RunEventKind.COMPLETED]), + run=edge_inventory, + default_status="stopped", +) +edge_factory_results.extend([edge_job_outcomes, edge_group_outcomes]) diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index 498b74207a..bd9320484f 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod attempts; pub(crate) mod facts; pub(crate) mod platform; pub(crate) mod requests; +pub(crate) mod runs; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs index 8215ce4379..208470a995 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -3,14 +3,16 @@ use uuid::Uuid; use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformEventKind; use crate::sensors::_helpers::requests::queue_sensor_run; -use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::sensors::models::{SensorCursor, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; pub(crate) async fn consume_platform_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, + kinds: &std::collections::BTreeSet, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { let (mut transaction, events) = read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; @@ -19,7 +21,6 @@ pub(crate) async fn consume_platform_events( .last() .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), }; - let SensorCondition::PlatformEvent { kinds } = &evaluation.definition.when; let mut run_ids: Vec = Vec::new(); for event in events .iter() diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs new file mode 100644 index 0000000000..1ac8ccaa38 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -0,0 +1,49 @@ +//! Terminal projections carry the durable finish cursor; late observations cannot move it. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{ + RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, +}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_run_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + job_key: &str, + kinds: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT run_id,status,last_event_storage_id FROM sqb_runs \ + WHERE location_id=$1 AND target=$2 AND job_key=$3 AND last_event_storage_id>$4 \ + AND status IN ('success','warn','partial','failed','cancelled','presumed_failed') \ + ORDER BY last_event_storage_id LIMIT 100", + vec![ + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + Parameter::Text(job_key.into()), + Parameter::Integer(evaluation.cursor.event_storage_id), + ], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let run: TerminalSensorRun = decode_row(row)?; + cursor.event_storage_id = run.last_event_storage_id; + if kinds.iter().any(|kind| kind.matches(run.status)) { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 5fcf9286c3..fc35a312c2 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -8,8 +8,9 @@ use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::_helpers::runs::consume_run_events; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; -use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -46,7 +47,14 @@ pub async fn complete_builtin_sensor( commit(transaction).await?; return Ok(0); } - let (transaction, (cursor, run_ids)) = consume_platform_events(transaction, evaluation).await?; + let (transaction, (cursor, run_ids)) = match &evaluation.definition.when { + SensorCondition::PlatformEvent { kinds } => { + consume_platform_events(transaction, evaluation, kinds).await? + } + SensorCondition::RunEvent { job_key, kinds } => { + consume_run_events(transaction, evaluation, job_key, kinds).await? + } + }; let count = run_ids.len(); let (transaction, (event_id, storage_id)) = append_evaluation( transaction, diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 06cd779823..60aee8f0ac 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -6,10 +6,17 @@ use crate::store::errors::ControlError; pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlError> { for (name, sensor) in &definitions.sensors { - let SensorCondition::PlatformEvent { kinds } = &sensor.when; + let valid_condition = match &sensor.when { + SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), + SensorCondition::RunEvent { job_key, kinds } => { + !kinds.is_empty() + && (definitions.jobs.contains_key(job_key) + || definitions.groups.contains_key(job_key)) + } + }; if name.trim().is_empty() || sensor.description.trim().is_empty() - || kinds.is_empty() + || !valid_condition || (!definitions.jobs.contains_key(&sensor.job_key) && !definitions.groups.contains_key(&sensor.job_key)) { diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index f39cf43d2f..9dff71a198 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -9,6 +9,7 @@ use uuid::Uuid; use crate::declarations::models::TriggerStatus; use crate::platform::models::PlatformEventKind; +use crate::runs::models::RunOutcome; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] @@ -31,7 +32,50 @@ impl SensorStatus { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum SensorCondition { - PlatformEvent { kinds: BTreeSet }, + PlatformEvent { + kinds: BTreeSet, + }, + RunEvent { + job_key: String, + kinds: BTreeSet, + }, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum RunEventKind { + Completed, + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunEventKind { + #[must_use] + pub const fn matches(self, outcome: RunOutcome) -> bool { + matches!( + (self, outcome), + (Self::Completed, _) + | (Self::Success, RunOutcome::Success) + | (Self::Warn, RunOutcome::Warn) + | (Self::Partial, RunOutcome::Partial) + | (Self::Failed, RunOutcome::Failed) + | (Self::Cancelled, RunOutcome::Cancelled) + | (Self::PresumedFailed, RunOutcome::PresumedFailed) + ) + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct TerminalSensorRun { + pub(crate) run_id: Uuid, + pub(crate) status: RunOutcome, + pub(crate) last_event_storage_id: i64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 5ae6a26780..560317b6ac 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -11,6 +11,7 @@ pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances. pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); +pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index dcc49f4793..59337ca99f 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,8 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, SCHEMA, SENSORS, render, verify_migrations, + LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, render, + verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -32,6 +33,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("location_work", LOCATION_WORK), ("platform_sink", PLATFORM_SINK), ("sensors", SENSORS), + ("run_sensors", RUN_SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/run_sensors.sql b/crates/sqlbuild-orchestration/src/store/run_sensors.sql new file mode 100644 index 0000000000..de46a16b7b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/run_sensors.sql @@ -0,0 +1,2 @@ +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled', 'presumed_failed'); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 4af6d90bef..19aa6d5e83 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 3f304e66c7..703b73d8e9 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -6,6 +6,14 @@ use crate::credentials::models::AuditContext; use crate::declarations::main::change_trigger::change_trigger; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; use crate::platform::models::PlatformEvent; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::models::SchedulerLease; use crate::sensors::models::SensorEvaluationFact; use crate::snapshots::models::SnapshotInput; use crate::store::main::append_control_event::append_control_event; @@ -128,3 +136,104 @@ pub(crate) async fn change(store: &ControlStore, trigger_id: Uuid, change: Trigg .expect("retry operator fact") ); } + +/// Real request, claim, node facts and finish, including the virtual group parent. +pub(crate) async fn finish_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + job_key: &str, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], +) -> Uuid { + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: scope.0, + target: scope.1.into(), + job_key: job_key.into(), + reason: "Exercise terminal sensor scope".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request source run"); + let limits = std::collections::BTreeMap::new(); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("advance parent"); + let claim = claim_next_run(store, lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("source ready"); + assert!(claim.request.run_id == run_id || claim.request.parent_run_id == Some(run_id)); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let changes = [ + RunChange::Started, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: nodes[0], + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: nodes[1], + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ]; + for change in changes { + change_run( + store, + worker, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("publish worker evidence"); + } + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome, + reason: "Terminal outcome fixture".into(), + exit_code: None, + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("finish") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("retry finish") + ); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("complete parent"); + run_id +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 04f4485f1e..b2e64838ac 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -6,5 +6,7 @@ mod helpers; mod test_delivery; #[path = "test_recovery.rs"] mod test_recovery; +#[path = "test_runs.rs"] +mod test_runs; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs new file mode 100644 index 0000000000..00cdef84b8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -0,0 +1,324 @@ +//! Actual lifecycle facts feed scoped outcome selectors and atomic recovery on both stores. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_run_keys_are_preserved() + { + let test_cases = [ + SensorTestCase { + description: "SQLite terminal outcome sensor recovery", + backend: Backend::Sqlite, + expected_runs: 16, + }, + SensorTestCase { + description: "PostgreSQL terminal outcome sensor recovery", + backend: Backend::Postgres, + expected_runs: 16, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store) + .await + .expect("warm migration preserves index"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}, "overlap": "allow"} + })); + let selectors = [ + "completed", + "success", + "warn", + "partial", + "failed", + "cancelled", + "presumed_failed", + ]; + let mut sensors = serde_json::Map::new(); + for job_key in ["orders", "daily_orders"] { + for selector in selectors { + sensors.insert(format!("{job_key}_{selector}"), json!({ + "description": "Consume exact terminal outcomes", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [selector]}, + "minimum_interval_seconds": 1 + })); + } + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input) + .await + .expect("publish sensors"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut expected: BTreeMap> = BTreeMap::new(); + let outcomes = [ + ( + "success", + RunOutcome::Success, + [NodeOutcome::Success, NodeOutcome::Success], + ), + ( + "warn", + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + ), + ( + "partial", + RunOutcome::Partial, + [NodeOutcome::Success, NodeOutcome::Failed], + ), + ( + "failed", + RunOutcome::Failed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ( + "cancelled", + RunOutcome::Cancelled, + [NodeOutcome::Cancelled, NodeOutcome::Cancelled], + ), + ( + "presumed_failed", + RunOutcome::PresumedFailed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ]; + for (selector, outcome, nodes) in outcomes { + let id = finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + ) + .await + .to_string(); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(id.clone()); + expected + .entry("orders_completed".into()) + .or_default() + .insert(id); + } + let group = finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + ) + .await; + let sources = list_runs(store, (location, "development"), 100) + .await + .expect("sources"); + let child = sources + .iter() + .find(|record| { + record.run_id != group + && record.job_key.as_deref() == Some("orders") + && !expected["orders_completed"].contains(&record.run_id.to_string()) + }) + .expect("group child"); + for selector in ["completed", "success"] { + expected + .entry(format!("daily_orders_{selector}")) + .or_default() + .insert(group.to_string()); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(child.run_id.to_string()); + } + // Same key in another target/location and a different job in this scope must not match. + let mut other_input = input.clone(); + other_input.location_id = other; + publish_snapshot(store, &other_input) + .await + .expect("other publication"); + other_input.location_id = location; + other_input.target = "preview".into(); + publish_snapshot(store, &other_input) + .await + .expect("other target"); + for scope in [(other, "development"), (location, "preview")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "inventory", + RunOutcome::Success, + [NodeOutcome::Success; 2], + ) + .await; + // Recreate the exact pre-change index/migration state with populated run history. + let transaction = begin(store, TransactionMode::Write) + .await + .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_runs_terminal_cursor", vec![]) + .await + .expect("remove new index"); + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_schema_migrations WHERE name='run_sensors'", + vec![], + ) + .await + .expect("remove new migration record"); + commit(transaction).await.expect("pre-change schema ready"); + migrate_store(store) + .await + .expect("upgrade populated prior schema"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "fail atomic sensor cursor", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + for trigger in &triggers { + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover"), + Some(requested.clone()) + ); + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &fault).await; + let wanted = expected.get(&trigger.name).cloned().unwrap_or_default(); + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry"), + wanted.len(), + "{}", + trigger.name + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("duplicate completion"), + 0 + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read requests"); + let (transaction, rows) = query( + transaction, + "SELECT run_key FROM sqb_run_requests WHERE trigger_id=$1", + vec![Parameter::Id(trigger.trigger_id)], + ) + .await + .expect("keys"); + commit(transaction).await.expect("read commit"); + let actual: BTreeSet = rows + .iter() + .map(|row| row["run_key"].as_str().expect("run key").into()) + .collect(); + assert_eq!(actual, wanted, "{}", trigger.name); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + } + tokio::time::sleep(Duration::from_secs(1)).await; + for trigger in &triggers { + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("reset request") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay deduplicates"); + } + let after = list_runs(store, (location, "development"), 100) + .await + .expect("queued sensors"); + assert_eq!( + after.len(), + sources.len() + 1 + test_case.expected_runs, + "{}: reset adds no runs", + test_case.description + ); + for source in sources { + assert_eq!( + after.iter().find(|record| record.run_id == source.run_id), + Some(&source), + "sensor never changes source state" + ); + } + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 5507104d12..2996933768 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ + "DROP INDEX sqb_runs_terminal_cursor; \ + DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 5c0e4cbe43..b7e704b8c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -2,3 +2,5 @@ /// The calendar's evaluation window is supplied by the host, not a provider. pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; + +pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 24a1f9f20e..0d9a84cfd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -10,6 +10,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use serde_json::Value; use sqlbuild_host::entries::models::UserCodeHost; +use crate::bindings::_helpers::native_host::constants::RUN_EVENT_CONDITION; use crate::bindings::_helpers::native_host::host::entry; #[derive(FromPyObject)] @@ -47,16 +48,18 @@ pub(crate) struct AuthoredGroup { } #[derive(FromPyObject)] -pub(crate) struct AuthoredPlatformCondition { +pub(crate) struct AuthoredSensorCondition { pub(crate) kind: String, pub(crate) kinds: Vec, + #[pyo3(default)] + pub(crate) job: Option, } #[derive(FromPyObject)] pub(crate) struct AuthoredSensor { pub(crate) name: String, pub(crate) description: String, - pub(crate) when: AuthoredPlatformCondition, + pub(crate) when: AuthoredSensorCondition, pub(crate) run: AuthoredJobReference, #[pyo3(from_py_with = authored_setting)] pub(crate) minimum_interval_seconds: Value, @@ -155,6 +158,16 @@ pub(crate) fn collect_declarations<'py>( )); } pending.push(target); + let condition = declaration.getattr("when")?; + if condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION { + let source = condition.getattr("job")?; + if !source.is_instance(&job)? && !source.is_instance(&group)? { + return Err(PyTypeError::new_err( + "run_event job must be a Job or JobGroup object", + )); + } + pending.push(source); + } AuthoredDeclaration::Sensor(declaration.extract()?) } else if declaration.is_instance(&calendar)? { let target = declaration.getattr("job")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index b787901fc2..0ce49660fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -263,10 +263,14 @@ pub(crate) fn compile_sensors( if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { continue; } + let mut condition = json!({"kind": sensor.when.kind, "kinds": sensor.when.kinds}); + if let Some(job) = sensor.when.job { + condition["job_key"] = json!(job.name); + } let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, "job_key": sensor.run.name, - "when": {"kind": sensor.when.kind, "kinds": sensor.when.kinds}, + "when": condition, "minimum_interval_seconds": sensor.minimum_interval_seconds, "default_status": sensor.default_status, })) diff --git a/python-boundary.toml b/python-boundary.toml index c5d7d33fc7..b50c3c0fc6 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -25,7 +25,7 @@ note = "Public job and schedule authoring records only; selection resolution, va [[stays]] path = "orchestration_declarations/types.py" rule = 3 -note = "Public PlatformEventKind enum only, placed in types.py to satisfy enum ownership; no Python event processing or implementation allowance." +note = "Public PlatformEventKind and RunEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." [[stays]] path = "orchestration_declarations/main/compile_snapshot.py" diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 0920f49d17..38fec4927a 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -7,6 +7,7 @@ Job, JobGroup, PlatformEventCondition, + RunEventCondition, Schedule, Sensor, ) @@ -14,9 +15,10 @@ from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event +from sqlbuild.orchestration_declarations.models import RunEventCondition as run_event from sqlbuild.orchestration_declarations.models import Schedule as schedule from sqlbuild.orchestration_declarations.models import Sensor as sensor -from sqlbuild.orchestration_declarations.types import PlatformEventKind +from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind __all__ = ( "CalendarSchedule", @@ -25,6 +27,8 @@ "JobGroup", "PlatformEventCondition", "PlatformEventKind", + "RunEventCondition", + "RunEventKind", "Schedule", "Sensor", "calendar_schedule", @@ -32,6 +36,7 @@ "job", "job_group", "platform_event", + "run_event", "schedule", "sensor", ) diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 8086e34bf6..97a4106dd0 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Literal -from sqlbuild.orchestration_declarations.types import PlatformEventKind +from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind @dataclass(frozen=True, kw_only=True) @@ -90,13 +90,22 @@ class PlatformEventCondition: kind: Literal["platform_event"] = field(default="platform_event", init=False) +@dataclass(frozen=True, kw_only=True) +class RunEventCondition: + """Match a job/group's terminal outcomes in the sensor's location and target.""" + + job: Job | JobGroup + kinds: Sequence[RunEventKind] + kind: Literal["run_event"] = field(default="run_event", init=False) + + @dataclass(frozen=True, eq=False) class Sensor: """Start a job for each matching event; event identity deduplicates replayed requests.""" name: str description: str = field(kw_only=True) - when: PlatformEventCondition = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition = field(kw_only=True) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index 913d56da06..ac8ef3e001 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -12,3 +12,15 @@ class PlatformEventKind(StrEnum): CALENDAR_HORIZON_LOW = "calendar_horizon_low" LOCATION_BROKEN = "location_broken" LAUNCH_FAILED = "launch_failed" + + +class RunEventKind(StrEnum): + """COMPLETED selects every terminal outcome; it is not a run status.""" + + COMPLETED = "completed" + SUCCESS = "success" + WARN = "warn" + PARTIAL = "partial" + FAILED = "failed" + CANCELLED = "cancelled" + PRESUMED_FAILED = "presumed_failed" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 87a99a43a4..c3169eea69 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field -from sqlbuild.orchestration import Schedule +from sqlbuild.orchestration import CalendarSchedule, Job, JobGroup, Schedule, Sensor @dataclass(frozen=True) @@ -251,7 +251,7 @@ class SnapshotDiscoveryTestCase: description: str files: dict[str, str] expected_jobs: tuple[str, ...] - declarations: tuple[()] | None = None + declarations: tuple[Job | JobGroup | CalendarSchedule | Sensor, ...] | None = None expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) expected_schedules: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index ae3edf17d5..f1f32da930 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -16,7 +16,7 @@ from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path -from typing import Any +from typing import Any, cast from urllib.error import HTTPError from urllib.parse import quote from urllib.request import Request, urlopen @@ -387,10 +387,16 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - "SELECT external_context FROM recall_invocations WHERE command = 'location'" ).fetchall() assert contexts, log.read_text() - assert all( - context is not None and json.loads(context)["run_id"] == run["run_id"] - for (context,) in contexts - ) + assert all(context is not None for (context,) in contexts) + expected_run_ids: set[str] = { + run_id + for (run_id,) in control.execute( + "SELECT run_id FROM sqb_runs WHERE kind<>'group' AND ended_at IS NOT NULL" + ).fetchall() + } + assert { + json.loads(cast(str, context))["run_id"] for (context,) in contexts + } == expected_run_ids assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 0812a7a2f5..c48b98bf0c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -122,8 +122,13 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical with urlopen(request, timeout=5) as response: triggers: list[dict[str, Any]] = json.load(response) by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} - assert len(triggers) == 2 - assert set(by_name) == {"edge_snapshot_orders", "edge_platform_orders"} + assert len(triggers) == 4 + assert set(by_name) == { + "edge_snapshot_orders", + "edge_platform_orders", + "edge_job_outcomes", + "edge_group_outcomes", + } scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] assert scheduled["status"] == "paused" assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" @@ -135,6 +140,18 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical "kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"], } + assert by_name["edge_job_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + } + assert by_name["edge_group_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + } + assert by_name["edge_job_outcomes"]["status"] == "paused" + assert by_name["edge_group_outcomes"]["status"] == "paused" @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index edae5b6127..8f46037730 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -487,6 +487,60 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group completion starts a real sensor worker", + "SELECT 59 AS amount", + "SELECT amount FROM orders", + "success", + [(59,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" + "on_group = sensor('on_group', description='Refresh after group completion.', " + "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS]), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ) + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destination_worker_executes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + result: dict[str, Any] = worker_result(worker, test_case) + assert result["run_id"] == destination["run_id"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT run_key,run_id FROM sqb_run_requests").fetchall() == [ + ( + control.execute( + "SELECT run_id FROM sqb_runs WHERE job_key='source_group'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 48ab859069..43cb006a6b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -18,7 +18,16 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration import Job, compile_snapshot, job, schedule +from sqlbuild.orchestration import ( + Job, + RunEventKind, + compile_snapshot, + job, + job_group, + run_event, + schedule, + sensor, +) from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, @@ -613,10 +622,66 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 7, "default_status": "stopped", - } + }, + "edge_job_outcomes": { + "description": "Keep warning and partial outcomes distinct from success.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + }, + "minimum_interval_seconds": 30, + "default_status": "stopped", + }, + "edge_group_outcomes": { + "description": "Observe group completion including cancellation and presumed failure.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + }, + "minimum_interval_seconds": 30, + "default_status": "stopped", + }, }, expected_schedules=("edge_snapshot_orders",), ), + SnapshotDiscoveryTestCase( + description="sensor-only roots collect source groups and destination jobs natively", + files={}, + declarations=( + sensor( + "on_orders", + description="Build inventory after group completion.", + when=run_event( + job=job_group( + "source_group", + description="Source orders.", + jobs=[job("source_orders", description="Orders.", select="orders")], + ), + kinds=[RunEventKind.COMPLETED], + ), + run=job("inventory", description="Inventory.", select="stg_orders"), + ), + ), + expected_jobs=("inventory", "source_orders"), + expected_groups={"source_group": {"source_orders": []}}, + expected_sensors={ + "on_orders": { + "description": "Build inventory after group completion.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_group", + "kinds": ["completed"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="sensor-only roots preserve referenced jobs and filter targets", files={ From 448a487914a841e7f40511169d104d4e00a8d771 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:37:23 +0000 Subject: [PATCH 052/138] feat: require successful executed nodes in run sensors --- .../edge_cases/orchestration/run_orders.py | 16 ++ .../orchestration/edge_cases/definitions.py | 6 +- .../src/sensors/_helpers/runs.rs | 56 ++++- .../src/sensors/constants.rs | 3 + .../sensors/main/complete_builtin_sensor.rs | 14 +- .../src/sensors/main/validate_sensors.rs | 19 +- .../sqlbuild-orchestration/src/sensors/mod.rs | 1 + .../src/sensors/models.rs | 2 + .../src/store/_helpers/migration.rs | 1 + .../src/store/executed_nodes.sql | 1 + .../src/store/main/migrate_store.rs | 5 +- .../tests/api/test_location_work.rs | 4 +- .../tests/sensors/helpers.rs | 44 ++-- .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_execution_bounds.rs | 176 ++++++++++++++ .../tests/sensors/test_runs.rs | 229 +++++++++++++++++- .../tests/sensors/test_types.rs | 10 + .../tests/snapshots/helpers.rs | 2 +- .../compiled_project/snapshot_declarations.rs | 1 + .../native_host/orchestration_declarations.rs | 2 + .../native_host/orchestration_triggers.rs | 16 ++ .../orchestration_declarations/models.py | 3 +- .../publication/test_calendar.py | 5 +- .../publication/test_publication.py | 1 + .../publication/test_worker.py | 2 +- .../test_orchestration_snapshot.py | 98 ++++++++ 26 files changed, 676 insertions(+), 43 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_orders.py create mode 100644 crates/sqlbuild-orchestration/src/sensors/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/store/executed_nodes.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py new file mode 100644 index 0000000000..bf6d376187 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py @@ -0,0 +1,16 @@ +"""Freeze source-job selectors without executing user code in the scheduler.""" + +from sqlbuild.orchestration import RunEventKind, job, run_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory only after orders actually succeeds.", + when=run_event( + job=source_orders, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders",), + ), + run=inventory, +) diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py index 8895ffef72..78b726a288 100644 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -86,7 +86,11 @@ edge_group_outcomes = sensor( "edge_group_outcomes", description="Observe group completion including cancellation and presumed failure.", - when=run_event(job=edge_group, kinds=[RunEventKind.COMPLETED]), + when=run_event( + job=edge_group, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("tag:edge_case",), + ), run=edge_inventory, default_status="stopped", ) diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs index 1ac8ccaa38..32b7e00e3c 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -5,6 +5,7 @@ use std::collections::BTreeSet; use uuid::Uuid; use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::constants::EXECUTION_NODE_BATCH_SIZE; use crate::sensors::models::{ RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, }; @@ -16,8 +17,8 @@ use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn consume_run_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, - job_key: &str, - kinds: &BTreeSet, + (job_key, kinds): (&str, &BTreeSet), + executed_successfully: Option<&BTreeSet>, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { let (mut transaction, rows) = query( transaction, @@ -33,12 +34,17 @@ pub(crate) async fn consume_run_events( ], ) .await?; + let runs: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let mut eligible: BTreeSet = runs.iter().map(|run| run.run_id).collect(); + if let Some(nodes) = executed_successfully { + (transaction, eligible) = matching_executions(transaction, eligible, nodes).await?; + } let mut cursor = evaluation.cursor.clone(); let mut run_ids: Vec = Vec::new(); - for row in rows { - let run: TerminalSensorRun = decode_row(row)?; + for run in runs { cursor.event_storage_id = run.last_event_storage_id; - if kinds.iter().any(|kind| kind.matches(run.status)) { + if eligible.contains(&run.run_id) && kinds.iter().any(|kind| kind.matches(run.status)) { let run_id; (transaction, run_id) = queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; @@ -47,3 +53,43 @@ pub(crate) async fn consume_run_events( } Ok((transaction, (cursor, run_ids))) } + +/// Each chunk intersects the previous matches: every selected node must qualify. +async fn matching_executions( + mut transaction: ControlTransaction, + mut eligible: BTreeSet, + nodes: &BTreeSet, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let nodes: Vec<&String> = nodes.iter().collect(); + for chunk in nodes.chunks(EXECUTION_NODE_BATCH_SIZE) { + if eligible.is_empty() { + break; + } + let mut parameters: Vec = eligible.iter().copied().map(Parameter::Id).collect(); + let run_parameters = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let node_parameters = (parameters.len() + 1..=parameters.len() + chunk.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + parameters.extend(chunk.iter().map(|key| Parameter::Text((*key).to_string()))); + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT r.run_id,r.status,r.last_event_storage_id FROM sqb_runs r \ + WHERE r.run_id IN ({run_parameters}) AND \ + (SELECT COUNT(*) FROM sqb_run_nodes n WHERE n.run_id=r.run_id AND n.attempt=1 \ + AND n.node_key IN ({node_parameters}) AND n.status='success' AND n.ended_at IS NOT NULL \ + AND ((r.kind<>'group' AND n.started_at IS NOT NULL) OR \ + (r.kind='group' AND EXISTS(SELECT 1 FROM sqb_runs c \ + JOIN sqb_run_nodes cn ON cn.run_id=c.run_id AND cn.node_key=n.node_key AND cn.attempt=1 \ + WHERE c.parent_run_id=r.run_id AND c.location_id=r.location_id AND c.target=r.target \ + AND cn.status='success' AND cn.started_at IS NOT NULL AND cn.ended_at IS NOT NULL))))={}", + chunk.len()), parameters).await?; + let matches: Vec = + rows.into_iter().map(decode_row).collect::>()?; + eligible = matches.into_iter().map(|run| run.run_id).collect(); + } + Ok((transaction, eligible)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs new file mode 100644 index 0000000000..e2b63365a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -0,0 +1,3 @@ +//! Bound query parameters independently of the size of an authored selection. + +pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index fc35a312c2..6a3640933e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -51,8 +51,18 @@ pub async fn complete_builtin_sensor( SensorCondition::PlatformEvent { kinds } => { consume_platform_events(transaction, evaluation, kinds).await? } - SensorCondition::RunEvent { job_key, kinds } => { - consume_run_events(transaction, evaluation, job_key, kinds).await? + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + consume_run_events( + transaction, + evaluation, + (job_key, kinds), + executed_successfully.as_ref(), + ) + .await? } }; let count = run_ids.len(); diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 60aee8f0ac..76b8bc2b75 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -8,10 +8,21 @@ pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlE for (name, sensor) in &definitions.sensors { let valid_condition = match &sensor.when { SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), - SensorCondition::RunEvent { job_key, kinds } => { + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { !kinds.is_empty() - && (definitions.jobs.contains_key(job_key) - || definitions.groups.contains_key(job_key)) + && definitions + .jobs + .get(job_key) + .or_else(|| definitions.groups.get(job_key).map(|group| &group.job)) + .is_some_and(|source| { + executed_successfully.as_ref().is_none_or(|nodes| { + !nodes.is_empty() && nodes.is_subset(&source.selection) + }) + }) } }; if name.trim().is_empty() @@ -21,7 +32,7 @@ pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlE && !definitions.groups.contains_key(&sensor.job_key)) { return Err(ControlError::Invalid(format!( - "sensor {name} needs a description, a known job and nonempty event kinds" + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes" ))); } } diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs index ca1039eddc..0f364e6e62 100644 --- a/crates/sqlbuild-orchestration/src/sensors/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -1,6 +1,7 @@ //! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 9dff71a198..5784a28c19 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -38,6 +38,8 @@ pub enum SensorCondition { RunEvent { job_key: String, kinds: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + executed_successfully: Option>, }, } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 560317b6ac..58b8897399 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -12,6 +12,7 @@ pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); +pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/executed_nodes.sql b/crates/sqlbuild-orchestration/src/store/executed_nodes.sql new file mode 100644 index 0000000000..4bfa512dad --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/executed_nodes.sql @@ -0,0 +1 @@ +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 59337ca99f..53a6d45481 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,8 +4,8 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, render, - verify_migrations, + EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, + render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -34,6 +34,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("platform_sink", PLATFORM_SINK), ("sensors", SENSORS), ("run_sensors", RUN_SENSORS), + ("executed_nodes", EXECUTED_NODES), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 19aa6d5e83..f52cad361a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 703b73d8e9..ff6089837f 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -145,6 +145,7 @@ pub(crate) async fn finish_run( job_key: &str, outcome: RunOutcome, nodes: [NodeOutcome; 2], + started: [bool; 2], ) -> Uuid { let run_id = Uuid::now_v7(); let context = AuditContext { @@ -168,6 +169,20 @@ pub(crate) async fn finish_run( ) .await .expect("request source run"); + finish_claimed_run(store, lease, scope, run_id, outcome, nodes, started).await; + run_id +} + +/// Execute only the frozen selection, including a from-failure rerun's reduced scope. +pub(crate) async fn finish_claimed_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + run_id: Uuid, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], + started: [bool; 2], +) { let limits = std::collections::BTreeMap::new(); advance_groups(store, lease, (&limits, scope)) .await @@ -181,23 +196,23 @@ pub(crate) async fn finish_run( run_id: claim.request.run_id, claim_id: claim.claim_id, }; - let changes = [ - RunChange::Started, - RunChange::NodeFinished { - node_key: "customers".into(), - outcome: nodes[0], - rows_affected: None, - error_code: None, - skipped_because_of: None, - }, - RunChange::NodeFinished { - node_key: "orders".into(), - outcome: nodes[1], + let mut changes = vec![RunChange::Started]; + for (index, key) in ["customers", "orders"] + .into_iter() + .enumerate() + .filter(|(_, key)| claim.request.selection.contains(*key)) + { + changes.extend(started[index].then_some(RunChange::NodeStarted { + node_key: key.into(), + })); + changes.push(RunChange::NodeFinished { + node_key: key.into(), + outcome: nodes[index], rows_affected: None, error_code: None, skipped_because_of: None, - }, - ]; + }); + } for change in changes { change_run( store, @@ -235,5 +250,4 @@ pub(crate) async fn finish_run( advance_groups(store, lease, (&limits, scope)) .await .expect("complete parent"); - run_id } diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index b2e64838ac..81c48ae068 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -4,6 +4,8 @@ mod helpers; #[path = "test_delivery.rs"] mod test_delivery; +#[path = "test_execution_bounds.rs"] +mod test_execution_bounds; #[path = "test_recovery.rs"] mod test_recovery; #[path = "test_runs.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs new file mode 100644 index 0000000000..3cc7469c46 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -0,0 +1,176 @@ +//! A wide group forces multiple predicate chunks; the last node is not optional. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::snapshot; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_must_match_within_a_bounded_evaluation() + { + let test_cases = [ + SensorTestCase { + description: "SQLite wide executed-node predicate", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL wide executed-node predicate", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let node = input.nodes["orders"].clone(); + let selection: Vec = (0..501) + .map(|index| format!("model:orders_{index:04}")) + .collect(); + input + .nodes + .extend(selection.iter().map(|key| (key.clone(), node.clone()))); + input.documents.insert("jobs".into(), json!({ + "source": {"description": "Wide source", "selection": selection, "overlap": "allow"}, + "destination": {"description": "Destination", "selection": ["orders"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "wide_orders": {"description": "Wide group", "selection": selection, "jobs": {"source": []}, "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "executed": {"description": "All source nodes must execute", "job_key": "destination", + "when": {"kind": "run_event", "job_key": "wide_orders", "kinds": ["completed"], "executed_successfully": selection}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input) + .await + .expect("publish wide predicate"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = std::collections::BTreeMap::new(); + let scope = (location, "development"); + for include_last in [true, false] { + let parent = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id: parent, + location_id: location, + target: "development".into(), + job_key: "wide_orders".into(), + reason: "Exercise wide execution evidence".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit parent"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("child"); + assert_eq!(claim.request.parent_run_id, Some(parent)); + let changes = std::iter::once(RunChange::Started) + .chain( + selection + .iter() + .filter(|key| include_last || key.as_str() != "model:orders_0500") + .map(|key| RunChange::NodeStarted { + node_key: key.clone(), + }), + ) + .chain(selection.iter().map(|key| RunChange::NodeFinished { + node_key: key.clone(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + })) + .chain(std::iter::once(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "All logical nodes succeed".into(), + exit_code: None, + })); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("real node evidence"); + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + } + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let evaluation = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let count = tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &evaluation), + ) + .await + .expect("wide predicate must not hold the write lock unboundedly") + .expect("complete"); + assert_eq!(count, test_case.expected_runs, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 00cdef84b8..7e3348ce3e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -6,18 +6,20 @@ use std::time::Duration; use serde_json::json; use uuid::Uuid; +use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; use crate::runs::main::list_runs::list_runs; -use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{NodeOutcome, RerunRequest, RunOutcome}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::{ begin_sensor_evaluation::begin_sensor_evaluation, complete_builtin_sensor::complete_builtin_sensor, }; -use crate::sensors::tests::helpers::{change, finish_run, snapshot}; -use crate::sensors::tests::test_types::SensorTestCase; +use crate::sensors::tests::helpers::{change, finish_claimed_run, finish_run, snapshot}; +use crate::sensors::tests::test_types::{SensorExecutionTestCase, SensorTestCase}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, @@ -36,12 +38,12 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r SensorTestCase { description: "SQLite terminal outcome sensor recovery", backend: Backend::Sqlite, - expected_runs: 16, + expected_runs: 106, }, SensorTestCase { description: "PostgreSQL terminal outcome sensor recovery", backend: Backend::Postgres, - expected_runs: 16, + expected_runs: 106, }, ]; for test_case in test_cases { @@ -86,6 +88,19 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "minimum_interval_seconds": 1 })); } + for (name, kind, nodes) in [ + ("both_completed", "completed", vec!["customers", "orders"]), + ("customers_completed", "completed", vec!["customers"]), + ("customers_success", "success", vec!["customers"]), + ("customers_partial", "partial", vec!["customers"]), + ("orders_completed", "completed", vec!["orders"]), + ] { + sensors.insert(format!("{job_key}_executed_{name}"), json!({ + "description": "Require every selected node to actually succeed", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [kind], "executed_successfully": nodes}, + "minimum_interval_seconds": 1 + })); + } } input.documents.insert("triggers".into(), json!(sensors)); publish_snapshot(store, &input) @@ -136,6 +151,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "orders", outcome, nodes, + [false; 2], ) .await .to_string(); @@ -155,6 +171,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "daily_orders", RunOutcome::Success, [NodeOutcome::Success; 2], + [false; 2], ) .await; let sources = list_runs(store, (location, "development"), 100) @@ -178,6 +195,199 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .or_default() .insert(child.run_id.to_string()); } + let executions = [ + SensorExecutionTestCase { + description: "freshness-only success cannot borrow another execution's starts", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [false; 2], + expected_selector: "success", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "all selected nodes must start, not merely any node", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true, false], + expected_selector: "success", + expected_filters: &["customers_completed", "customers_success"], + }, + SensorExecutionTestCase { + description: "both selected nodes actually succeed", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true; 2], + expected_selector: "success", + expected_filters: &[ + "both_completed", + "customers_completed", + "customers_success", + "orders_completed", + ], + }, + SensorExecutionTestCase { + description: "partial outcome is independent of selected-node success", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [true; 2], + expected_selector: "partial", + expected_filters: &["customers_completed", "customers_partial"], + }, + SensorExecutionTestCase { + description: "warning is not successful execution", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true; 2], + expected_selector: "warn", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "start without success cannot borrow success from another run", + outcome: RunOutcome::Failed, + nodes: [NodeOutcome::Failed; 2], + started: [true; 2], + expected_selector: "failed", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "skipped selected node is not successful", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Skipped, NodeOutcome::Success], + started: [false, true], + expected_selector: "partial", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "a partial run's freshness-only node still does not qualify", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [false, true], + expected_selector: "partial", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "a genuinely started warning is not success", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true, false], + expected_selector: "warn", + expected_filters: &[], + }, + ]; + let mut source_ids: BTreeMap<(&str, &str), Uuid> = BTreeMap::new(); + for execution in executions { + for job_key in ["orders", "daily_orders"] { + let id = finish_run( + store, + &lease, + (location, "development"), + job_key, + execution.outcome, + execution.nodes, + execution.started, + ) + .await; + source_ids.insert((execution.description, job_key), id); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(id)], + ) + .await + .expect(execution.description); + commit(transaction).await.expect("read lineage committed"); + for (source, id) in std::iter::once((job_key, id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for kind in ["completed", execution.expected_selector] { + expected + .entry(format!("{source}_{kind}")) + .or_default() + .insert(id.clone()); + } + for filter in execution.expected_filters { + expected + .entry(format!("{source}_executed_{filter}")) + .or_default() + .insert(id.clone()); + } + } + } + } + let retry = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_ids[&( + "partial outcome is independent of selected-node success", + "daily_orders", + )], + from_failure: true, + reason: "Retry only failed orders without borrowing earlier customer success" + .into(), + priority: 10, + }; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!( + request_rerun(store, &context, &retry) + .await + .expect("request rerun") + ); + assert!( + !request_rerun(store, &context, &retry) + .await + .expect("retry request") + ); + finish_claimed_run( + store, + &lease, + (location, "development"), + retry.run_id, + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read rerun lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(retry.run_id)], + ) + .await + .expect("rerun child"); + commit(transaction).await.expect("read rerun committed"); + for (source, id) in std::iter::once(("daily_orders", retry.run_id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for suffix in ["completed", "success", "executed_orders_completed"] { + expected + .entry(format!("{source}_{suffix}")) + .or_default() + .insert(id.clone()); + } + } + let sources = list_runs(store, (location, "development"), 500) + .await + .expect("all source executions"); // Same key in another target/location and a different job in this scope must not match. let mut other_input = input.clone(); other_input.location_id = other; @@ -197,6 +407,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "orders", RunOutcome::Success, [NodeOutcome::Success; 2], + [false; 2], ) .await; } @@ -207,19 +418,23 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r "inventory", RunOutcome::Success, [NodeOutcome::Success; 2], + [false; 2], ) .await; // Recreate the exact pre-change index/migration state with populated run history. let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_parent", vec![]) + .await + .expect("remove execution lookup index"); let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_terminal_cursor", vec![]) .await .expect("remove new index"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name='run_sensors'", + "DELETE FROM sqb_schema_migrations WHERE version>=6", vec![], ) .await @@ -301,7 +516,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .await .expect("replay deduplicates"); } - let after = list_runs(store, (location, "development"), 100) + let after = list_runs(store, (location, "development"), 500) .await .expect("queued sensors"); assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs index b762f2f4c5..1b4e85322d 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -1,3 +1,4 @@ +use crate::runs::models::{NodeOutcome, RunOutcome}; use crate::store::tests::store_test_types::Backend; pub(crate) struct SensorTestCase { @@ -5,3 +6,12 @@ pub(crate) struct SensorTestCase { pub(crate) backend: Backend, pub(crate) expected_runs: usize, } + +pub(crate) struct SensorExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: RunOutcome, + pub(crate) nodes: [NodeOutcome; 2], + pub(crate) started: [bool; 2], + pub(crate) expected_selector: &'static str, + pub(crate) expected_filters: &'static [&'static str], +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 2996933768..4faee91009 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP INDEX sqb_runs_terminal_cursor; \ + "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index be7fa9ac3b..caab6c25e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -203,6 +203,7 @@ pub(crate) fn compile_declarations( sensors, } = compile_sensors( &input.target, + graph, authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 0d9a84cfd8..87db7962f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -53,6 +53,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) kinds: Vec, #[pyo3(default)] pub(crate) job: Option, + #[pyo3(default)] + pub(crate) executed_successfully: Option>, } #[derive(FromPyObject)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 0ce49660fa..f3ba7c955e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -10,6 +10,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; @@ -249,6 +251,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( target: &str, + graph: &ProjectGraph, authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -267,6 +270,19 @@ pub(crate) fn compile_sensors( if let Some(job) = sensor.when.job { condition["job_key"] = json!(job.name); } + if let Some(selectors) = sensor.when.executed_successfully { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err( + "executed_successfully needs nonempty selectors", + )); + } + let nodes: BTreeSet = resolve_selectors(graph, &selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + condition["executed_successfully"] = json!(nodes); + } let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, "job_key": sensor.run.name, diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 97a4106dd0..077fde537f 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -92,10 +92,11 @@ class PlatformEventCondition: @dataclass(frozen=True, kw_only=True) class RunEventCondition: - """Match a job/group's terminal outcomes in the sensor's location and target.""" + """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" job: Job | JobGroup kinds: Sequence[RunEventKind] + executed_successfully: Sequence[str] | None = None kind: Literal["run_event"] = field(default="run_event", init=False) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 7eb241a5fe..52fabe8463 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -147,7 +147,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" ).read_text(), "providers/fulfillment.py": ( - "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta\n" + "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta, timezone\n" "from sqlbuild.provider.classes.provider import Provider\n" "class Fulfillment(Provider):\n" ' """Upcoming synthetic fulfillment times."""\n' @@ -160,7 +160,8 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" - " return [window.start + timedelta(seconds=offset) for offset in data['offsets']]\n" + " start = window.start.astimezone(timezone.utc)\n" + " return [start + timedelta(seconds=offset) for offset in data['offsets']]\n" ), }, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index c48b98bf0c..9b12212a01 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -149,6 +149,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical "kind": "run_event", "job_key": "edge_snapshot_group", "kinds": ["completed"], + "executed_successfully": ["model:orders"], } assert by_name["edge_job_outcomes"]["status"] == "paused" assert by_name["edge_group_outcomes"]["status"] == "paused" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 8f46037730..5e04e48389 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -505,7 +505,7 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" "on_group = sensor('on_group', description='Refresh after group completion.', " - "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS]), " + "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " "run=annual_orders, minimum_interval_seconds=1)\n" ) }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 43cb006a6b..c4804832b3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -641,6 +641,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "kind": "run_event", "job_key": "edge_snapshot_group", "kinds": ["completed"], + "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "stopped", @@ -662,6 +663,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject jobs=[job("source_orders", description="Orders.", select="orders")], ), kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders", "orders*"), ), run=job("inventory", description="Inventory.", select="stg_orders"), ), @@ -676,6 +678,30 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "kind": "run_event", "job_key": "source_group", "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="source-job execution selectors compile to frozen canonical keys", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory only after orders actually succeeds.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_orders", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "running", @@ -858,6 +884,78 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="unknown variant", ), + SnapshotDiscoveryRejectionTestCase( + description="explicit empty execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', "()") + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="blank execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '(" ",)') + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="unresolved execution selectors fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_orders",)') + }, + expected_error=ValueError, + expected_message="unknown selector name 'unknown_orders'", + ), + SnapshotDiscoveryRejectionTestCase( + description="unmatched execution patterns cannot become an omitted filter", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_*",)') + }, + expected_error=ValueError, + expected_message="unknown selector pattern", + ), + SnapshotDiscoveryRejectionTestCase( + description="invalid execution selector syntax fails closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("tag:",)') + }, + expected_error=ValueError, + expected_message="has empty value after ':'", + ), + SnapshotDiscoveryRejectionTestCase( + description="execution selectors outside the source job fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("stg_orders",)') + }, + expected_error=ValueError, + expected_message="executed_successfully must select nonempty source job/group nodes", + ), SnapshotDiscoveryRejectionTestCase( description="sensor code defaults use running or stopped", files={ From d72827412ab1f40e434f826b0ac37dd34e5f0203 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:57:13 +0000 Subject: [PATCH 053/138] feat: trigger sensors for genuine successful node rebuilds Resolve authored selectors natively and consume immutable node finishes with earlier same-run and claim start evidence. Keep per-finish deduplication inside the fenced sensor transaction and exclude freshness-only results and group summaries. Extend only the existing enum declaration boundary for NodeEventKind, retaining a zero Python implementation budget. --- .../edge_cases/orchestration/node_orders.py | 12 + .../declarations/main/snapshot_definitions.rs | 2 +- .../src/sensors/_helpers/conditions.rs | 37 ++ .../src/sensors/_helpers/mod.rs | 2 + .../src/sensors/_helpers/nodes.rs | 95 +++++ .../sensors/main/complete_builtin_sensor.rs | 24 +- .../src/sensors/main/validate_sensors.rs | 13 +- .../src/sensors/models.rs | 20 + .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/node_sensors.sql | 6 + .../tests/api/test_location_work.rs | 4 +- .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_nodes.rs | 376 ++++++++++++++++++ .../tests/sensors/test_runs.rs | 8 + .../tests/snapshots/helpers.rs | 3 +- .../native_host/orchestration_declarations.rs | 2 + .../native_host/orchestration_triggers.rs | 36 +- python-boundary.toml | 2 +- src/sqlbuild/orchestration.py | 7 +- .../orchestration_declarations/models.py | 13 +- .../orchestration_declarations/types.py | 6 + .../publication/test_worker.py | 59 +++ .../test_orchestration_snapshot.py | 46 +++ 24 files changed, 737 insertions(+), 44 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_orders.py create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/store/node_sensors.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py new file mode 100644 index 0000000000..ac8f6cb8fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py @@ -0,0 +1,12 @@ +"""Trigger on each real orders rebuild, not on freshness-only success.""" + +from sqlbuild.orchestration import NodeEventKind, job, node_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run=inventory, +) diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index cbe7c7a152..0e288368d8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -81,6 +81,6 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result {} } } - validate_sensors(&definitions)?; + validate_sensors(&definitions, input)?; Ok(definitions) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs new file mode 100644 index 0000000000..6e2e9f5d1a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -0,0 +1,37 @@ +//! Each native condition consumes its bounded page inside the evaluation transaction. + +use uuid::Uuid; + +use crate::sensors::_helpers::nodes::consume_node_events; +use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::_helpers::runs::consume_run_events; +use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_condition( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + match &evaluation.definition.when { + SensorCondition::PlatformEvent { kinds } => { + consume_platform_events(transaction, evaluation, kinds).await + } + SensorCondition::NodeEvent { selection, .. } => { + consume_node_events(transaction, evaluation, selection).await + } + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + consume_run_events( + transaction, + evaluation, + (job_key, kinds), + executed_successfully.as_ref(), + ) + .await + } + } +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index bd9320484f..267996a120 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod attempts; +pub(crate) mod conditions; pub(crate) mod facts; +pub(crate) mod nodes; pub(crate) mod platform; pub(crate) mod requests; pub(crate) mod runs; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs new file mode 100644 index 0000000000..c7f7a97b92 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -0,0 +1,95 @@ +//! Successful rebuilds require earlier immutable start evidence from the same run claim. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{NodeSensorEvent, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_node_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT storage_id,event_id,run_id,payload FROM recall_events \ + WHERE producer='scheduler' AND event_type='run_node_finished' \ + AND invocation_id='scheduler/' || run_id AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", + vec![Parameter::Integer(evaluation.cursor.event_storage_id)], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let event: NodeSensorEvent = decode_row(row)?; + cursor.event_storage_id = event.storage_id; + let request: RunChangeRequest = serde_json::from_value(event.payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.run_id != event.run_id || request.event_id != event.event_id { + return Err(ControlError::Invalid("node event identity mismatch".into())); + } + let RunChange::NodeFinished { + node_key, + outcome: NodeOutcome::Success, + .. + } = &request.change + else { + continue; + }; + if !selection.contains(node_key) { + continue; + } + let started; + (transaction, started) = + prior_start(transaction, evaluation, &request, event.storage_id).await?; + if started { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} + +async fn prior_start( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + request: &RunChangeRequest, + storage_id: i64, +) -> Result<(ControlTransaction, bool), ControlError> { + let (RunChange::NodeFinished { node_key, .. }, Some(claim_id)) = + (&request.change, request.claim_id) + else { + return Ok((transaction, false)); + }; + let (transaction, rows) = query( + transaction, + "SELECT s.event_id FROM sqb_runs r JOIN recall_events s ON s.run_id=CAST(r.run_id AS TEXT) \ + WHERE s.invocation_id=$1 AND s.run_id=$2 AND s.producer='scheduler' \ + AND s.event_type='run_node_started' AND s.storage_id<$3 \ + AND (s.payload -> 'change' ->> 'claim_id')=$4 \ + AND (s.payload -> 'change' -> 'change' ->> 'node_key')=$5 \ + AND r.run_id=$6 AND r.location_id=$7 AND r.target=$8 AND r.kind<>'group' LIMIT 1", + vec![ + Parameter::Text(format!("scheduler/{}", request.run_id)), + Parameter::Text(request.run_id.to_string()), + Parameter::Integer(storage_id), + Parameter::Text(claim_id.to_string()), + Parameter::Text(node_key.clone()), + Parameter::Id(request.run_id), + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + ], + ) + .await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 6a3640933e..8982ae3db5 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -6,11 +6,10 @@ use crate::declarations::models::TriggerStatus; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::conditions::consume_condition; use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; -use crate::sensors::_helpers::platform::consume_platform_events; -use crate::sensors::_helpers::runs::consume_run_events; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; -use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -47,24 +46,7 @@ pub async fn complete_builtin_sensor( commit(transaction).await?; return Ok(0); } - let (transaction, (cursor, run_ids)) = match &evaluation.definition.when { - SensorCondition::PlatformEvent { kinds } => { - consume_platform_events(transaction, evaluation, kinds).await? - } - SensorCondition::RunEvent { - job_key, - kinds, - executed_successfully, - } => { - consume_run_events( - transaction, - evaluation, - (job_key, kinds), - executed_successfully.as_ref(), - ) - .await? - } - }; + let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; let count = run_ids.len(); let (transaction, (event_id, storage_id)) = append_evaluation( transaction, diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 76b8bc2b75..0459f426dc 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -2,12 +2,21 @@ use crate::declarations::models::Definitions; use crate::sensors::models::SensorCondition; +use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; -pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlError> { +pub(crate) fn validate_sensors( + definitions: &Definitions, + input: &SnapshotInput, +) -> Result<(), ControlError> { for (name, sensor) in &definitions.sensors { let valid_condition = match &sensor.when { SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), + SensorCondition::NodeEvent { selection, kinds } => { + !kinds.is_empty() + && !selection.is_empty() + && selection.iter().all(|key| input.nodes.contains_key(key)) + } SensorCondition::RunEvent { job_key, kinds, @@ -32,7 +41,7 @@ pub(crate) fn validate_sensors(definitions: &Definitions) -> Result<(), ControlE && !definitions.groups.contains_key(&sensor.job_key)) { return Err(ControlError::Invalid(format!( - "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes" + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes" ))); } } diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 5784a28c19..5fa409329d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -41,6 +41,18 @@ pub enum SensorCondition { #[serde(default, skip_serializing_if = "Option::is_none")] executed_successfully: Option>, }, + NodeEvent { + selection: BTreeSet, + kinds: BTreeSet, + }, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum NodeEventKind { + Success, } #[derive( @@ -80,6 +92,14 @@ pub(crate) struct TerminalSensorRun { pub(crate) last_event_storage_id: i64, } +#[derive(Debug, Deserialize)] +pub(crate) struct NodeSensorEvent { + pub(crate) event_id: Uuid, + pub(crate) run_id: Uuid, + pub(crate) storage_id: i64, + pub(crate) payload: serde_json::Value, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct SensorDefinition { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 58b8897399..b2c9b5fbf4 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -13,6 +13,7 @@ pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); +pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 53a6d45481..6190e2798a 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,8 +4,8 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, - render, verify_migrations, + EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, + SCHEMA, SENSORS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -35,6 +35,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("sensors", SENSORS), ("run_sensors", RUN_SENSORS), ("executed_nodes", EXECUTED_NODES), + ("node_sensors", NODE_SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/node_sensors.sql b/crates/sqlbuild-orchestration/src/store/node_sensors.sql new file mode 100644 index 0000000000..372ccf6b4c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/node_sensors.sql @@ -0,0 +1,6 @@ +CREATE INDEX sqb_node_finished_cursor ON recall_events(storage_id) +WHERE producer='scheduler' AND event_type='run_node_finished'; +CREATE INDEX sqb_node_started_identity ON recall_events( + invocation_id, (payload -> 'change' ->> 'claim_id'), + (payload -> 'change' -> 'change' ->> 'node_key'), storage_id +) WHERE producer='scheduler' AND event_type='run_node_started'; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index f52cad361a..bc54a92560 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,9 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 81c48ae068..e6429a8404 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -6,6 +6,8 @@ mod helpers; mod test_delivery; #[path = "test_execution_bounds.rs"] mod test_execution_bounds; +#[path = "test_nodes.rs"] +mod test_nodes; #[path = "test_recovery.rs"] mod test_recovery; #[path = "test_runs.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs new file mode 100644 index 0000000000..bc2556174c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -0,0 +1,376 @@ +//! Real lifecycle facts exercise any-run rebuilds, early delivery and paged atomic recovery. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queue_once_per_finish() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite any-run node successes", + backend: Backend::Sqlite, + expected_runs: 8, + }, + SensorTestCase { + description: "PostgreSQL any-run node successes", + backend: Backend::Postgres, + expected_runs: 8, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}, "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "rebuilt_orders": {"description": "Each genuine rebuild", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["success"]}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input).await.expect("publish"); + let mut isolated = input.clone(); + isolated.location_id = other; + publish_snapshot(store, &isolated) + .await + .expect("other location snapshot"); + isolated.location_id = location; + isolated.target = "staging".into(); + publish_snapshot(store, &isolated) + .await + .expect("other target snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + for (outcome, nodes, started) in [ + (RunOutcome::Success, [NodeOutcome::Success; 2], [true; 2]), + (RunOutcome::Success, [NodeOutcome::Success; 2], [false; 2]), + ( + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + [true; 2], + ), + (RunOutcome::Failed, [NodeOutcome::Failed; 2], [true; 2]), + ( + RunOutcome::Partial, + [NodeOutcome::Skipped, NodeOutcome::Success], + [false, true], + ), + ( + RunOutcome::Cancelled, + [NodeOutcome::Cancelled; 2], + [true; 2], + ), + ] { + finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + started, + ) + .await; + } + for scope in [(other, "development"), (location, "staging")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + // Page past nonmatches without borrowing starts from earlier real runs. + for _ in 0..46 { + finish_run( + store, + &lease, + (location, "development"), + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + } + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: location, + target: "development".into(), + job_key: "orders".into(), + reason: "Deliver before whole-run completion".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request running source"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("ready"); + assert_eq!(claim.request.run_id, run_id); + let authority = RunAuthority::Worker { + run_id, + claim_id: claim.claim_id, + }; + let changes = [ + RunChange::Started, + RunChange::NodeStarted { + node_key: "orders".into(), + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ]; + for change in changes { + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }; + let mut wrong = request.clone(); + wrong.claim_id = Some(Uuid::now_v7()); + assert!( + change_run(store, authority, &wrong).await.is_err(), + "another claim cannot supply start/finish evidence" + ); + assert!( + change_run(store, authority, &request) + .await + .expect("worker fact") + ); + assert!( + !change_run(store, authority, &request) + .await + .expect("duplicate worker fact") + ); + } + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "node cursor rollback", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected node cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected node cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &pending) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rollback") + .iter() + .all(|run| run.job_key.as_deref() != Some("inventory")) + ); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(pending.clone()) + ); + assert_eq!( + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(&replica, &lease, &pending), + ) + .await + .expect("bounded 100-event page") + .expect("first page"), + 6 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &pending) + .await + .expect("duplicate completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let next = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("second evaluation") + .expect("due"); + assert!(next.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &next) + .await + .expect("second page"), + 1, + "the other selected node is still pending" + ); + for change in [ + RunChange::NodeStarted { + node_key: "customers".into(), + }, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("later node in the same run"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + let later = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("later evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &later) + .await + .expect("later finish"), + 1, + "distinct successful finishes in the same run must not collapse" + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("runs"); + assert_eq!( + runs.iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == run_id) + .expect("source retained") + .status, + crate::runs::models::RunStatus::Running + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + for _ in 0..2 { + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("deduplicated replay"); + } + assert_eq!( + list_runs(store, (location, "development"), 1000) + .await + .expect("replayed runs") + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 7e3348ce3e..545f2373f9 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,14 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_finished_cursor", vec![]) + .await + .expect("remove node finish cursor index"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_started_identity", vec![]) + .await + .expect("remove node start evidence index"); let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_parent", vec![]) .await .expect("remove execution lookup index"); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 4faee91009..3464b72c6f 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ + "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ + DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 87db7962f8..453a9f5145 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -55,6 +55,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) job: Option, #[pyo3(default)] pub(crate) executed_successfully: Option>, + #[pyo3(default)] + pub(crate) select: Option, } #[derive(FromPyObject)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index f3ba7c955e..03f6bb4e77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -270,18 +270,13 @@ pub(crate) fn compile_sensors( if let Some(job) = sensor.when.job { condition["job_key"] = json!(job.name); } + if let Some(select) = sensor.when.select { + let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; + } if let Some(selectors) = sensor.when.executed_successfully { - if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { - return Err(PyValueError::new_err( - "executed_successfully needs nonempty selectors", - )); - } - let nodes: BTreeSet = resolve_selectors(graph, &selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); - condition["executed_successfully"] = json!(nodes); + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; } let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, @@ -295,3 +290,22 @@ pub(crate) fn compile_sensors( } Ok(definitions) } + +fn sensor_nodes(graph: &ProjectGraph, selectors: &[String], field: &str) -> PyResult { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + let nodes: BTreeSet = resolve_selectors(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if nodes.is_empty() { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + Ok(json!(nodes)) +} diff --git a/python-boundary.toml b/python-boundary.toml index b50c3c0fc6..159acb56e3 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -25,7 +25,7 @@ note = "Public job and schedule authoring records only; selection resolution, va [[stays]] path = "orchestration_declarations/types.py" rule = 3 -note = "Public PlatformEventKind and RunEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." +note = "Public PlatformEventKind, RunEventKind and NodeEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." [[stays]] path = "orchestration_declarations/main/compile_snapshot.py" diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 38fec4927a..3532cf5f7d 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -6,6 +6,7 @@ CalendarWindow, Job, JobGroup, + NodeEventCondition, PlatformEventCondition, RunEventCondition, Schedule, @@ -14,17 +15,20 @@ from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule from sqlbuild.orchestration_declarations.models import Job as job from sqlbuild.orchestration_declarations.models import JobGroup as job_group +from sqlbuild.orchestration_declarations.models import NodeEventCondition as node_event from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event from sqlbuild.orchestration_declarations.models import RunEventCondition as run_event from sqlbuild.orchestration_declarations.models import Schedule as schedule from sqlbuild.orchestration_declarations.models import Sensor as sensor -from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind __all__ = ( "CalendarSchedule", "CalendarWindow", "Job", "JobGroup", + "NodeEventCondition", + "NodeEventKind", "PlatformEventCondition", "PlatformEventKind", "RunEventCondition", @@ -35,6 +39,7 @@ "compile_snapshot", "job", "job_group", + "node_event", "platform_event", "run_event", "schedule", diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models.py index 077fde537f..8f4a44a9b8 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Literal -from sqlbuild.orchestration_declarations.types import PlatformEventKind, RunEventKind +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind @dataclass(frozen=True, kw_only=True) @@ -100,13 +100,22 @@ class RunEventCondition: kind: Literal["run_event"] = field(default="run_event", init=False) +@dataclass(frozen=True, kw_only=True) +class NodeEventCondition: + """Match each successful rebuild in any run within the sensor's location and target.""" + + select: str + kinds: Sequence[NodeEventKind] + kind: Literal["node_event"] = field(default="node_event", init=False) + + @dataclass(frozen=True, eq=False) class Sensor: """Start a job for each matching event; event identity deduplicates replayed requests.""" name: str description: str = field(kw_only=True) - when: PlatformEventCondition | RunEventCondition = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition | NodeEventCondition = field(kw_only=True) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index ac8ef3e001..4635074185 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -24,3 +24,9 @@ class RunEventKind(StrEnum): FAILED = "failed" CANCELLED = "cancelled" PRESUMED_FAILED = "presumed_failed" + + +class NodeEventKind(StrEnum): + """SUCCESS requires a genuine rebuild, not freshness-only completion.""" + + SUCCESS = "success" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 5e04e48389..391f828b68 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -541,6 +541,65 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin ] +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group child node sensor", + "SELECT 67 AS amount", + "SELECT amount FROM orders", + "success", + [(67,)], + expected_origin="sensor", + files={ + "models/stg_orders.sql": "MODEL (description 'Staging orders.');\nSELECT 71 AS amount\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" + "on_node = sensor('on_node', description='React to actual orders rebuilds.', " + "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_child_rebuild_when_a_node_sensor_matches_then_only_the_child_finish_queues_a_worker( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + assert control.execute( + "SELECT q.run_key,q.run_id FROM sqb_run_requests q" + ).fetchall() == [ + ( + control.execute( + "SELECT e.event_id FROM recall_events e JOIN sqb_runs r ON r.run_id=e.run_id " + "WHERE r.job_key='source_orders' AND e.event_type='run_node_finished'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index c4804832b3..ded89951dd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -685,6 +685,28 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="any-run node selectors freeze canonical keys without a source job reference", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["success"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", files={ @@ -884,6 +906,30 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="unknown variant", ), + SnapshotDiscoveryRejectionTestCase( + description="empty node selectors cannot broaden matching to all nodes", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace('select="orders orders*"', 'select=" "') + }, + expected_error=ValueError, + expected_message="node_event needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="node event kinds do not imply unapproved failure transitions", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['failed']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", files={ From bab7e7c68298da5fb199b605027bb1d85ff1cce3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:59:18 +0000 Subject: [PATCH 054/138] feat: execute custom polling sensors in isolated locations Commit proposed cursors and deduplicated requests only through fenced atomic completion. Defer pending evaluations fairly, preserve the public authoring imports, and isolate the approved mutable context declaration with a structural scope guard. Adopt Fensu 0.28.3 for declaration-role recognition. Preserve optional bare-command dispatch and confine adversarial terminal width to the location child process. --- .../edge_cases/orchestration/custom_orders.py | 15 + .../src/native_commands/constants.rs | 3 + .../native_commands/main/_location_sensor.rs | 76 +++ .../native_commands/main/_location_work.rs | 32 +- .../src/native_commands/main/mod.rs | 1 + crates/sqlbuild-orchestration/src/api/mod.rs | 2 +- .../src/platform/_helpers/location_work.rs | 17 - .../src/platform/main/claim_location_work.rs | 13 +- .../platform/main/complete_location_work.rs | 30 +- .../src/platform/main/location_work.rs | 4 +- .../src/platform/main/mod.rs | 1 + .../src/platform/main/read_location_work.rs | 24 + .../platform/main/request_location_work.rs | 11 +- .../src/platform/models.rs | 6 + .../src/scheduler/_helpers/drain_sensors.rs | 8 +- .../src/scheduler/tests.rs | 2 + .../src/scheduler/tests/test_types.rs | 9 + .../src/sensors/_helpers/completion.rs | 77 +++ .../src/sensors/_helpers/conditions.rs | 3 + .../src/sensors/_helpers/custom.rs | 104 ++++ .../src/sensors/_helpers/mod.rs | 2 + .../src/sensors/_helpers/platform.rs | 1 + .../src/sensors/constants.rs | 2 + .../sensors/main/begin_sensor_evaluation.rs | 2 + .../sensors/main/complete_builtin_sensor.rs | 59 +-- .../sensors/main/complete_custom_sensor.rs | 42 ++ .../src/sensors/main/mod.rs | 1 + .../src/sensors/main/validate_sensors.rs | 4 + .../sqlbuild-orchestration/src/sensors/mod.rs | 4 +- .../src/sensors/models.rs | 22 + .../tests/api/test_contracts.rs | 2 +- .../tests/scheduler/test_sensor_fairness.rs | 128 +++++ .../tests/sensors/test_contracts.rs | 4 +- .../tests/sensors/test_custom.rs | 443 ++++++++++++++++++ .../tests/sensors/test_types.rs | 7 + .../src/bindings/_helpers/cli/entry.rs | 10 +- .../compiled_project/snapshot_declarations.rs | 4 +- .../_helpers/location_command/runtime.rs | 68 ++- .../_helpers/native_host/constants.rs | 3 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_declarations.rs | 7 +- .../native_host/orchestration_environment.rs | 60 ++- .../native_host/orchestration_sensors.rs | 132 ++++++ .../native_host/orchestration_triggers.rs | 49 +- fensu.toml | 6 + pyproject.toml | 2 +- python-boundary.toml | 7 +- src/sqlbuild/orchestration.py | 28 +- .../main/compile_snapshot.py | 7 +- .../models/__init__.py | 0 .../{models.py => models/authoring.py} | 24 +- .../models/sensor_context.py | 12 + .../compiled_project/publication/helpers.py | 31 +- .../publication/test_custom_sensor.py | 160 +++++++ .../publication/test_reload.py | 7 +- .../test_orchestration_snapshot.py | 36 ++ uv.lock | 30 +- 57 files changed, 1679 insertions(+), 166 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/custom_orders.py create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs create mode 100644 crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_custom.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs create mode 100644 src/sqlbuild/orchestration_declarations/models/__init__.py rename src/sqlbuild/orchestration_declarations/{models.py => models/authoring.py} (87%) create mode 100644 src/sqlbuild/orchestration_declarations/models/sensor_context.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py new file mode 100644 index 0000000000..f448f2b85f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py @@ -0,0 +1,15 @@ +"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" + +from sqlbuild.orchestration import job, sensor + +orders = job("annual_orders", description="Build orders after source inspection.", select="orders") + + +@sensor( + "poll_orders", description="Poll fulfillment readiness.", run=orders, minimum_interval_seconds=1 +) +def poll_orders(ctx, fulfillment): + def poll(): + return fulfillment.poll(ctx) + + return poll() diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 74c0358e56..65784a5e9b 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -90,3 +90,6 @@ pub(crate) const EXIT_FLUSH_FAILURE: &str = "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; +/// Reserve half the location work claim for acknowledgement retries after killing a callback. +pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; +pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs new file mode 100644 index 0000000000..86c4585092 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs @@ -0,0 +1,76 @@ +//! A callback gets its own interpreter; timeout and shutdown reap it before reporting a result. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::process::Child; + +use crate::native_commands::_helpers::execution::location_runtime_child; +use crate::native_commands::constants::{SENSOR_EVALUATION_SECONDS, SENSOR_RESULT_BYTES}; +use crate::native_commands::models::LocationServe; + +pub(super) async fn evaluate_location_sensor( + options: &LocationServe, + script: &Path, + evaluation: &SensorEvaluationRequest, + stopped: impl Future>, +) -> Result, ControlError> { + let request = + serde_json::to_vec(evaluation).map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut child = match location_runtime_child(options, script) { + Ok(child) => child, + Err(_) => return Ok(Some(CustomSensorOutcome::Failed)), + }; + tokio::pin!(stopped); + let outcome = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = tokio::time::timeout(Duration::from_secs(SENSOR_EVALUATION_SECONDS), exchange_result(&mut child, request)) => { + match result { + Ok(Ok((_, outcome))) => outcome, + Ok(Err(_)) => CustomSensorOutcome::Failed, + Err(_) => CustomSensorOutcome::TimedOut, + } + }, + }; + if child.try_wait()?.is_none() { + child.kill().await?; + } + Ok(Some(outcome)) +} + +/// A bounded reply is accepted only after its interpreter exits successfully. +async fn exchange_result( + child: &mut Child, + request: Vec, +) -> Result<(&mut Child, CustomSensorOutcome), ControlError> { + let mut stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("sensor input is unavailable".into()))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("sensor output is unavailable".into()))?; + stdin.write_all(&request).await?; + stdin.write_all(b"\n").await?; + stdin.shutdown().await?; + drop(stdin); + let mut bytes: Vec = Vec::new(); + let mut stdout = stdout.take(SENSOR_RESULT_BYTES + 1); + stdout.read_to_end(&mut bytes).await?; + if bytes.len() as u64 > SENSOR_RESULT_BYTES || !child.wait().await?.success() { + return Ok((child, CustomSensorOutcome::Failed)); + } + Ok(( + child, + serde_json::from_slice(&bytes).unwrap_or(CustomSensorOutcome::Failed), + )) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 281b8bb686..abf575eb77 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -16,6 +16,7 @@ use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{LocationServe, LocationWatch}; /// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. @@ -44,11 +45,14 @@ pub(super) async fn run_location_work( .to_std() .unwrap_or_default(); let mut files: Vec = Vec::new(); - let outcome = if generation == Some(claim.generation) { - drop(write_output(&[( - Stream::Stderr, - "Checking requested location connection...\n", - )])); + let outcome = if generation == Some(claim.generation) + && location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed + { + let message = match &claim.request.work { + LocationWork::CheckConnection {} => "Checking requested location connection...\n", + LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + }; + drop(write_output(&[(Stream::Stderr, message)])); let mut shutting_down = false; let expires = async { tokio::select! { @@ -57,11 +61,21 @@ pub(super) async fn run_location_work( } }; let checked = match claim.request.work { - LocationWork::CheckConnection {} => { - check_location_connection(options, script, expires).await? + LocationWork::CheckConnection {} => check_location_connection(options, script, expires) + .await? + .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + LocationWork::EvaluateSensor { evaluation } => { + evaluate_location_sensor(options, script, &evaluation, expires) + .await? + .map(|outcome| { + ( + LocationWorkOutcome::SensorEvaluation { outcome }, + Vec::new(), + ) + }) } }; - let Some((healthy, local_files)) = checked else { + let Some((outcome, local_files)) = checked else { drop(write_output(&[( Stream::Stderr, "Location work stopped before completion; no result reported.\n", @@ -76,7 +90,7 @@ pub(super) async fn run_location_work( let mut excluded = watch.local_files.clone(); excluded.extend(files.clone()); if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { - LocationWorkOutcome::Connection { healthy } + outcome } else { LocationWorkOutcome::Superseded {} } diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index a1333f51a6..af7c252502 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,5 +1,6 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; +pub(super) mod _location_sensor; pub(super) mod _location_work; pub(super) mod _run_admin; pub(super) mod _run_control; diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index cb776dbca6..524d320a8c 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -8,4 +8,4 @@ pub mod models; #[cfg(test)] #[path = "../../tests/api/test_contracts.rs"] -mod tests; +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs index c523d62210..8e2d429a5a 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs @@ -2,27 +2,10 @@ use uuid::Uuid; -use crate::platform::models::StoredLocationWork; use crate::store::errors::ControlError; -use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -pub(crate) async fn read_work( - transaction: ControlTransaction, - request_id: Uuid, -) -> Result<(ControlTransaction, Option), ControlError> { - let (transaction, rows) = query(transaction, - "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ - w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ - FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ - WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; - Ok(( - transaction, - rows.into_iter().next().map(decode_row).transpose()?, - )) -} - pub(crate) async fn require_instance( transaction: ControlTransaction, (location_id, session_id, target): (Uuid, Uuid, &str), diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs index fd24042528..7a36796c37 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -3,7 +3,8 @@ use chrono::Duration; use uuid::Uuid; -use crate::platform::_helpers::location_work::{read_work, require_instance}; +use crate::platform::_helpers::location_work::require_instance; +use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -26,6 +27,14 @@ pub async fn claim_location_work( "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ AND w.expires_at>$3 AND NOT EXISTS (SELECT 1 FROM sqb_location_work_results r WHERE r.request_id=w.request_id) \ AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ + AND (w.request_json->'work'->>'kind'='check_connection' OR EXISTS ( \ + SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ + JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ + WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ + AND t.snapshot_id=w.snapshot_id AND t.removed_in_snapshot_id IS NULL AND l.expires_at>$3 \ + AND l.epoch=CAST(e.fact_json->'request'->>'epoch' AS BIGINT) \ + AND NOT EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events f \ + WHERE f.evaluation_id=e.evaluation_id AND f.phase<>'requested'))) \ ORDER BY w.requested_at, w.request_id LIMIT 1", vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; let Some(row) = rows.first() else { @@ -34,7 +43,7 @@ pub async fn claim_location_work( }; let request_id = serde_json::from_value(row["request_id"].clone()) .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; - let (transaction, stored) = read_work(transaction, request_id).await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; let retained = stored.session_id == Some(poll.session_id) && stored.claimed_until.is_some_and(|until| until > now); diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index 0f1888ed8b..a044885fe4 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -3,8 +3,11 @@ use serde_json::json; use uuid::Uuid; -use crate::platform::_helpers::location_work::{read_work, require_instance}; -use crate::platform::models::LocationWorkResult; +use crate::platform::_helpers::location_work::require_instance; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; +use crate::sensors::constants::MAX_SENSOR_RUN_REQUESTS; +use crate::sensors::models::CustomSensorOutcome; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -17,7 +20,7 @@ pub async fn complete_location_work( result: &LocationWorkResult, ) -> Result { let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, stored) = read_work(transaction, request_id).await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored .filter(|stored| stored.record.request_json.location_id == location_id) .ok_or_else(|| ControlError::Invalid("work is not assigned to this location".into()))?; @@ -48,6 +51,27 @@ pub async fn complete_location_work( { return Err(ControlError::Fenced); } + let valid = match (&stored.record.request_json.work, &result.outcome) { + (_, LocationWorkOutcome::Superseded {}) + | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + ( + LocationWork::EvaluateSensor { .. }, + LocationWorkOutcome::SensorEvaluation { outcome }, + ) => match outcome { + CustomSensorOutcome::Completed { run_keys, .. } => { + run_keys.len() <= MAX_SENSOR_RUN_REQUESTS + && run_keys.iter().all(|key| !key.trim().is_empty()) + } + CustomSensorOutcome::Skipped { reason, .. } => !reason.trim().is_empty(), + CustomSensorOutcome::Failed | CustomSensorOutcome::TimedOut => true, + }, + _ => false, + }; + if !valid { + return Err(ControlError::Invalid( + "location result does not match its work".into(), + )); + } let now = transaction.now; let (transaction, _) = query(transaction, "INSERT INTO sqb_location_work_results (request_id,result_json,completed_at) VALUES ($1,$2,$3)", diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs index f71f848d79..8a8783c56f 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::platform::_helpers::location_work::read_work; +use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::LocationWorkRecord; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -14,7 +14,7 @@ pub async fn location_work( request_id: Uuid, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, record) = read_work(transaction, request_id).await?; + let (transaction, record) = read_location_work(transaction, request_id).await?; commit(transaction).await?; Ok(record.map(|stored| stored.record)) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 850f9e7c04..a782c64402 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -4,6 +4,7 @@ pub mod connect_location_work; pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; +pub(crate) mod read_location_work; pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs new file mode 100644 index 0000000000..8401a196a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs @@ -0,0 +1,24 @@ +//! Read durable work inside a caller's existing transaction, including fenced sensor completion. + +use uuid::Uuid; + +use crate::platform::models::StoredLocationWork; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_location_work( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ + w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ + FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ + WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index d03b92dce4..df0c73f23d 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -6,8 +6,8 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::platform::_helpers::location_work::read_work; -use crate::platform::models::LocationWorkRequest; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -19,6 +19,11 @@ pub async fn request_location_work( context: &AuditContext, request: &LocationWorkRequest, ) -> Result { + if matches!(request.work, LocationWork::EvaluateSensor { .. }) { + return Err(ControlError::Invalid( + "sensor work must originate from a fenced evaluation".into(), + )); + } if request.target.trim().is_empty() || request.reason.trim().is_empty() || context.actor.trim().is_empty() @@ -28,7 +33,7 @@ pub async fn request_location_work( )); } let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = read_work(transaction, request.request_id).await?; + let (transaction, previous) = read_location_work(transaction, request.request_id).await?; if let Some(previous) = previous { if previous.record.request_json != *request || previous.record.requested_by != context.actor { diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 7ab2762ac9..fab9c20eab 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; /// One pooled outbound connection; the location never receives a store handle. #[derive(Clone, Debug)] @@ -45,6 +46,10 @@ pub struct LocationHealthObservation { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWork { CheckConnection {}, + EvaluateSensor { + #[schema(value_type = Object)] + evaluation: Box, + }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -78,6 +83,7 @@ pub struct LocationWorkClaim { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWorkOutcome { Connection { healthy: bool }, + SensorEvaluation { outcome: CustomSensorOutcome }, Superseded {}, } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs index 2456383857..964ee871b6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs @@ -4,6 +4,8 @@ use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::SensorCondition; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -27,7 +29,11 @@ pub(crate) async fn drain_sensors( let trigger_id = serde_json::from_value(row["trigger_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; if let Some(request) = begin_sensor_evaluation(store, &lease, trigger_id).await? { - complete_builtin_sensor(store, &lease, &request).await?; + if matches!(request.definition.when, SensorCondition::Custom { .. }) { + complete_custom_sensor(store, &lease, &request).await?; + } else { + complete_builtin_sensor(store, &lease, &request).await?; + } evaluated += 1; } lease = renew_lease(store, &lease).await?; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs index 927f476ce9..3a3124f551 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -6,6 +6,8 @@ mod helpers; mod test_backlog; #[path = "tests/test_cron.rs"] mod test_cron; +#[path = "../../tests/scheduler/test_sensor_fairness.rs"] +mod test_sensor_fairness; #[path = "tests/test_simulation.rs"] mod test_simulation; #[path = "tests/test_ticks.rs"] diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs index ee92508cab..ab3d443e4f 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -1,6 +1,15 @@ //! Inputs make cursor, catch-up, and overlap decisions observably different. use crate::scheduler::models::{CatchUp, Overlap}; +use crate::store::tests::store_test_types::Backend; + +pub(super) struct SensorFairnessTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) pending: usize, + pub(super) builtin_status: &'static str, + pub(super) expected_completed: usize, +} #[derive(Debug)] pub(super) struct TickTestCase { diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs new file mode 100644 index 0000000000..20964ded97 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs @@ -0,0 +1,77 @@ +//! Every evaluation result uses the same lease, snapshot, cursor and terminal boundary. + +use chrono::TimeDelta; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn pending_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + if evaluation.epoch != lease.epoch { + return Err(ControlError::Fenced); + } + let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; + let Some((previous, terminal)) = previous else { + return Err(ControlError::Fenced); + }; + if previous.request != *evaluation { + return Err(ControlError::Fenced); + } + if terminal { + commit(transaction).await?; + return Ok(None); + } + let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; + let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; + if sensor.is_none_or(|sensor| { + sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id + }) || cursor != evaluation.cursor + { + let (transaction, _) = + append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; + commit(transaction).await?; + return Ok(None); + } + Ok(Some(transaction)) +} + +pub(crate) async fn finish_evaluation( + transaction: ControlTransaction, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(), ControlError> { + let cursor = match &outcome { + SensorEvaluationOutcome::Completed { cursor, .. } => Some(cursor.clone()), + _ => None, + }; + let (mut transaction, (event_id, storage_id)) = + append_evaluation(transaction, evaluation, outcome).await?; + if let Some(cursor) = cursor { + transaction = commit_cursor( + transaction, + (evaluation.trigger_id, event_id, storage_id), + &cursor, + ) + .await?; + } + let next = evaluation.requested_at + + TimeDelta::seconds(i64::from( + evaluation.definition.minimum_interval_seconds.get(), + )); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs index 6e2e9f5d1a..2e4243170e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -14,6 +14,9 @@ pub(crate) async fn consume_condition( evaluation: &SensorEvaluationRequest, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { match &evaluation.definition.when { + SensorCondition::Custom { .. } => Err(ControlError::Invalid( + "custom sensors require a location result".into(), + )), SensorCondition::PlatformEvent { kinds } => { consume_platform_events(transaction, evaluation, kinds).await } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs new file mode 100644 index 0000000000..c2e65bb427 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs @@ -0,0 +1,104 @@ +//! Custom callbacks cross only the existing location-scoped outbound-work boundary. + +use std::collections::BTreeSet; + +use chrono::TimeDelta; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn request_evaluation_work( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Ok(transaction); + } + let request = LocationWorkRequest { + request_id: evaluation.evaluation_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + reason: "Evaluate custom sensor".into(), + work: LocationWork::EvaluateSensor { + evaluation: Box::new(evaluation.clone()), + }, + }; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + SELECT $1,$2,$3,snapshot_id,generation,$4,'system:sensor',$5,$6 FROM sqb_snapshots WHERE snapshot_id=$7", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Json(json!(request)), Parameter::Time(evaluation.requested_at), + Parameter::Time(evaluation.requested_at + TimeDelta::minutes(5)), Parameter::Id(evaluation.snapshot_id)]).await?; + Ok(transaction) +} + +pub(crate) async fn read_evaluation_result( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, work) = read_location_work(transaction, evaluation.evaluation_id).await?; + let work = work.ok_or_else(|| ControlError::Invalid("sensor has no location work".into()))?; + if work.record.result_json.is_none() { + let expired = work.record.expires_at <= transaction.now; + return Ok(( + transaction, + expired.then_some(SensorEvaluationOutcome::Abandoned), + )); + } + match work.record.result_json.map(|result| result.outcome) { + Some(LocationWorkOutcome::SensorEvaluation { outcome }) => { + apply_result(transaction, evaluation, outcome).await + } + Some(LocationWorkOutcome::Superseded {}) => { + Ok((transaction, Some(SensorEvaluationOutcome::Abandoned))) + } + _ => Err(ControlError::Invalid( + "sensor has an invalid location result".into(), + )), + } +} + +async fn apply_result( + mut transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + outcome: CustomSensorOutcome, +) -> Result<(ControlTransaction, Option), ControlError> { + let (custom, run_keys) = match outcome { + CustomSensorOutcome::Completed { cursor, run_keys } => (cursor, run_keys), + CustomSensorOutcome::Skipped { cursor, .. } => (cursor, Vec::new()), + CustomSensorOutcome::Failed => { + return Ok(( + transaction, + Some(SensorEvaluationOutcome::Failed { + code: "custom_sensor_failed".into(), + }), + )); + } + CustomSensorOutcome::TimedOut => { + return Ok((transaction, Some(SensorEvaluationOutcome::TimedOut))); + } + }; + let mut cursor = evaluation.cursor.clone(); + cursor.custom = custom; + let mut run_ids: Vec = Vec::new(); + let run_keys: BTreeSet = run_keys.into_iter().collect(); + for key in run_keys { + let run_id; + (transaction, run_id) = queue_sensor_run(transaction, evaluation, &key).await?; + run_ids.push(run_id); + } + Ok(( + transaction, + Some(SensorEvaluationOutcome::Completed { cursor, run_ids }), + )) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index 267996a120..720f46154e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod attempts; +pub(crate) mod completion; pub(crate) mod conditions; +pub(crate) mod custom; pub(crate) mod facts; pub(crate) mod nodes; pub(crate) mod platform; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs index 208470a995..6ce610bf03 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -20,6 +20,7 @@ pub(crate) async fn consume_platform_events( event_storage_id: events .last() .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), + ..evaluation.cursor.clone() }; let mut run_ids: Vec = Vec::new(); for event in events diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs index e2b63365a5..c74670d166 100644 --- a/crates/sqlbuild-orchestration/src/sensors/constants.rs +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -1,3 +1,5 @@ //! Bound query parameters independently of the size of an authored selection. pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; +pub(crate) const CUSTOM_SENSOR_RECHECK_SECONDS: i64 = 1; +pub const MAX_SENSOR_RUN_REQUESTS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs index 45b82e4cbb..db7c36bb49 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -8,6 +8,7 @@ use crate::declarations::models::TriggerStatus; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::custom::request_evaluation_work; use crate::sensors::_helpers::facts::append_evaluation; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; @@ -75,6 +76,7 @@ pub async fn begin_sensor_evaluation( }; let (transaction, _) = append_evaluation(transaction, &request, SensorEvaluationOutcome::Requested).await?; + let transaction = request_evaluation_work(transaction, &request).await?; commit(check_lease(transaction, lease).await?).await?; Ok(Some(request)) } diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 8982ae3db5..5fbd35834d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -1,73 +1,28 @@ //! Cursor, outcome, deduplicated requests and queued runs commit together. -use chrono::TimeDelta; - -use crate::declarations::models::TriggerStatus; -use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; -use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; use crate::sensors::_helpers::conditions::consume_condition; -use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; -use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::ControlStore; pub async fn complete_builtin_sensor( store: &ControlStore, lease: &SchedulerLease, evaluation: &SensorEvaluationRequest, ) -> Result { - let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; - if evaluation.epoch != lease.epoch { - return Err(ControlError::Fenced); - } - let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; - let Some((previous, terminal)) = previous else { - return Err(ControlError::Fenced); - }; - if previous.request != *evaluation { - return Err(ControlError::Fenced); - } - if terminal { - commit(transaction).await?; + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { return Ok(0); - } - let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; - let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; - if sensor.is_none_or(|sensor| { - sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id - }) || cursor != evaluation.cursor - { - let (transaction, _) = - append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; - commit(transaction).await?; - return Ok(0); - } + }; let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; let count = run_ids.len(); - let (transaction, (event_id, storage_id)) = append_evaluation( + finish_evaluation( transaction, + lease, evaluation, - SensorEvaluationOutcome::Completed { - cursor: cursor.clone(), - run_ids, - }, - ) - .await?; - let transaction = commit_cursor( - transaction, - (evaluation.trigger_id, event_id, storage_id), - &cursor, + SensorEvaluationOutcome::Completed { cursor, run_ids }, ) .await?; - let next = evaluation.requested_at - + TimeDelta::seconds(i64::from( - evaluation.definition.minimum_interval_seconds.get(), - )); - let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; - commit(check_lease(transaction, lease).await?).await?; Ok(count) } diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs new file mode 100644 index 0000000000..98d23dd900 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs @@ -0,0 +1,42 @@ +//! A location result cannot queue runs or advance a cursor until a fenced scheduler commits it. + +use chrono::TimeDelta; + +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::defer_sensor; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; +use crate::sensors::_helpers::custom::read_evaluation_result; +use crate::sensors::constants::CUSTOM_SENSOR_RECHECK_SECONDS; +use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; + +pub async fn complete_custom_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Err(ControlError::Invalid( + "custom sensor evaluation required".into(), + )); + } + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { + return Ok(0); + }; + let (transaction, outcome) = read_evaluation_result(transaction, evaluation).await?; + let Some(outcome) = outcome else { + let next = transaction.now + TimeDelta::seconds(CUSTOM_SENSOR_RECHECK_SECONDS); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(0); + }; + let count = match &outcome { + SensorEvaluationOutcome::Completed { run_ids, .. } => run_ids.len(), + _ => 0, + }; + finish_evaluation(transaction, lease, evaluation, outcome).await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs index a57c8e3058..e819eb77e1 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -1,5 +1,6 @@ pub mod begin_sensor_evaluation; pub mod complete_builtin_sensor; +pub mod complete_custom_sensor; pub(crate) mod publish_sensors; pub(crate) mod read_sensor_cursor; pub(crate) mod reset_sensor_cursor; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 0459f426dc..da488cc75d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -11,6 +11,10 @@ pub(crate) fn validate_sensors( ) -> Result<(), ControlError> { for (name, sensor) in &definitions.sensors { let valid_condition = match &sensor.when { + SensorCondition::Custom { + function, + code_hash, + } => !function.trim().is_empty() && !code_hash.is_empty(), SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), SensorCondition::NodeEvent { selection, kinds } => { !kinds.is_empty() diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs index 0f364e6e62..9580a2dd16 100644 --- a/crates/sqlbuild-orchestration/src/sensors/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -1,10 +1,10 @@ //! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; #[cfg(test)] #[path = "../../tests/sensors/test_contracts.rs"] -mod tests; +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 5fa409329d..38fbfced25 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -45,6 +45,10 @@ pub enum SensorCondition { selection: BTreeSet, kinds: BTreeSet, }, + Custom { + function: String, + code_hash: String, + }, } #[derive( @@ -117,6 +121,24 @@ pub struct SensorDefinition { #[serde(deny_unknown_fields)] pub struct SensorCursor { pub event_storage_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, +} + +/// Location-owned Python returns data; only the fenced scheduler may apply it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum CustomSensorOutcome { + Completed { + cursor: Option, + run_keys: Vec, + }, + Skipped { + cursor: Option, + reason: String, + }, + Failed, + TimedOut, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index d141d28f37..722e726650 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -1,7 +1,7 @@ //! HTTP contracts compiled within the API owner. #[path = "helpers.rs"] -mod helpers; +pub(crate) mod helpers; #[path = "output/helpers.rs"] mod output_helpers; #[path = "recall/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs new file mode 100644 index 0000000000..5259d9ee0b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs @@ -0,0 +1,128 @@ +//! Unavailable callback locations cannot monopolize the shared due-sensor page. + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::test_types::SensorFairnessTestCase; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_pending_custom_sensors_when_scheduling_then_rechecks_wait_and_other_locations_progress() { + let test_cases = [ + SensorFairnessTestCase { + description: "SQLite pending callback waits", + backend: Backend::Sqlite, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "PostgreSQL pending callback waits", + backend: Backend::Postgres, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "SQLite unavailable callback page permits builtin progress", + backend: Backend::Sqlite, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + SensorFairnessTestCase { + description: "PostgreSQL unavailable callback page permits builtin progress", + backend: Backend::Postgres, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate fairness store"); + let custom = create_location(store, "fulfillment", "test-admin") + .await + .expect("callback location"); + let builtin = create_location(store, "reporting", "test-admin") + .await + .expect("builtin location"); + let mut input = snapshot(custom); + input.documents.insert("triggers".into(), json!({})); + for index in 0..test_case.pending { + input.documents.get_mut("triggers").expect("triggers")[format!("orders_{index}")] = json!({ + "description": "Poll fulfillment", "job_key": "orders", "when": {"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}, + "minimum_interval_seconds": 1 + }); + } + publish_snapshot(store, &input) + .await + .expect("publish unavailable callbacks"); + publish_snapshot(store, &snapshot(builtin)) + .await + .expect("publish independent builtin"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("set due order"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$2)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(2)), Parameter::Id(custom)]).await.expect("callbacks first"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,status=$2 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$3)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(1)), Parameter::Text(test_case.builtin_status.into()), Parameter::Id(builtin)]).await.expect("builtin last"); + commit(transaction).await.expect("commit due order"); + let (state, first) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("pending page"); + assert_eq!( + first.evaluated, test_case.pending, + "{}", + test_case.description + ); + assert_eq!( + first.wait_for.is_zero(), + test_case.expected_completed == 1, + "{}: only genuinely overdue unprocessed work should require an immediate retry", + test_case.description + ); + scheduler_cycle(store, state).await.expect("next page"); + let facts = evaluations(store).await; + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome == SensorEvaluationOutcome::Requested) + .count(), + test_case.pending + test_case.expected_completed, + "{}", + test_case.description + ); + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome != SensorEvaluationOutcome::Requested) + .count(), + test_case.expected_completed, + "{}: builtin at another location completes without waiting for unavailable callbacks", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index e6429a8404..84e211d230 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -1,7 +1,9 @@ //! Real stores prove atomic sensor consumption and recovery across scheduler ownership changes. #[path = "helpers.rs"] -mod helpers; +pub(crate) mod helpers; +#[path = "test_custom.rs"] +mod test_custom; #[path = "test_delivery.rs"] mod test_delivery; #[path = "test_execution_bounds.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs new file mode 100644 index 0000000000..e3407c4419 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -0,0 +1,443 @@ +//! The real outbound HTTP boundary returns data, never scheduler authority. + +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkResult, +}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationOutcome}; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::sensors::tests::test_types::{CustomSensorResultCase, SensorTestCase}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_custom_sensor_work_when_results_retry_then_cursor_runs_and_outcome_commit_atomically() { + let test_cases = [ + SensorTestCase { + description: "SQLite custom sensor protocol", + backend: Backend::Sqlite, + expected_runs: 2, + }, + SensorTestCase { + description: "PostgreSQL custom sensor protocol", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = + json!({"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}); + publish_snapshot(store, &input) + .await + .expect("publish custom sensor data"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &context, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + TimeDelta::hours(1), + ) + .await + .expect("scoped token"); + let api = start_api(store).await; + let client = + connect_location_work((&api.url, &token.bearer), location).expect("client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: "development".into(), + }; + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register location instance"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let point = TransactionFault { + description: "request and outbound work are atomic", + table: "sqb_location_work", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor request failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor request failure'", + }; + install_fault(store, &point).await; + assert!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + evaluations(store).await.is_empty(), + "request fact rolled back with work" + ); + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("begin") + .expect("due"); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("still waiting"), + 0 + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "no invented terminal phase while waiting" + ); + let claim = client.poll(&poll).await.expect("poll").expect("work"); + assert_eq!(claim.request.request_id, requested.evaluation_id); + assert_eq!( + claim.request.work, + LocationWork::EvaluateSensor { + evaluation: Box::new(requested.clone()) + } + ); + assert_eq!( + client.poll(&poll).await.expect("lost poll response"), + Some(claim.clone()) + ); + let wrong = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: true }, + }; + assert!( + client + .complete(requested.evaluation_id, &wrong) + .await + .is_err(), + "wrong result kind" + ); + let empty_key = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("unsafe".into()), + run_keys: vec![" ".into()], + }, + }, + ..wrong.clone() + }; + assert!( + client + .complete(requested.evaluation_id, &empty_key) + .await + .is_err(), + "empty keys fail closed" + ); + let result = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into(), "orders-2".into()], + }, + }, + ..wrong + }; + assert!( + client + .complete(requested.evaluation_id, &result) + .await + .expect("durable location result") + ); + assert!( + !client + .complete(requested.evaluation_id, &result) + .await + .expect("uncertain acknowledgement retry") + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .is_empty(), + "location result alone has no queue authority" + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + "sqb_sensor_cursor_commits", + ] { + let point = TransactionFault { + description: "custom sensor atomic publication", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_custom_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback runs") + .is_empty() + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "outcome rolled back with cursor and queue" + ); + assert_eq!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("unchanged cursor"), + Some(requested.clone()) + ); + } + let replica = second_store(store).await; + assert_eq!( + complete_custom_sensor(&replica, &lease, &requested) + .await + .expect("replica commit"), + test_case.expected_runs + ); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("retry commit"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("dedup runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + let facts = evaluations(store).await; + let SensorEvaluationOutcome::Completed { cursor, run_ids } = + &facts.last().expect("completed").outcome + else { + panic!("completed phase required"); + }; + assert_eq!(cursor.custom.as_deref(), Some("orders/page-2")); + assert_eq!(cursor.event_storage_id, requested.cursor.event_storage_id); + assert_eq!(run_ids.len(), test_case.expected_runs); + for result_case in [ + CustomSensorResultCase { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into()], + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-2"), + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Failed, + expected_phase: "failed", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::TimedOut, + expected_phase: "timed_out", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Skipped { + cursor: Some("orders/page-3".into()), + reason: "No new orders".into(), + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-3"), + }, + ] { + tokio::time::sleep(Duration::from_secs(1)).await; + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next attempt") + .expect("due"); + assert_eq!( + pending.cursor.custom.as_deref(), + Some("orders/page-2"), + "failed and timeout leave cursor unchanged" + ); + let claim = client + .poll(&poll) + .await + .expect("next poll") + .expect("next work"); + assert_eq!(claim.request.request_id, pending.evaluation_id); + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: result_case.outcome, + }, + }; + assert!( + client + .complete(pending.evaluation_id, &result) + .await + .expect("complete callback") + ); + complete_custom_sensor(store, &lease, &pending) + .await + .expect("apply phase"); + let facts = evaluations(store).await; + let outcome = &facts.last().expect("terminal evaluation phase").outcome; + assert_eq!(outcome.phase(), result_case.expected_phase); + let payload = serde_json::to_value(outcome).expect("outcome payload"); + assert_eq!( + payload["cursor"]["custom"].as_str(), + result_case.expected_cursor + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("still deduped") + .len(), + test_case.expected_runs + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let stale = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request before takeover") + .expect("due"); + let stale_claim = client + .poll(&poll) + .await + .expect("claim before takeover") + .expect("work"); + assert_eq!(stale_claim.request.request_id, stale.evaluation_id); + expire_lease(store).await; + assert!( + client + .poll(&poll) + .await + .expect("no expired-epoch dispatch") + .is_none() + ); + assert!(matches!( + complete_custom_sensor(store, &lease, &stale).await, + Err(ControlError::Fenced) + )); + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("leader"); + let late = LocationWorkResult { + claim_id: stale_claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("uncommitted/stale".into()), + run_keys: vec!["must-not-queue".into()], + }, + }, + }; + assert!( + client + .complete(stale.evaluation_id, &late) + .await + .expect("late location result") + ); + assert!(matches!( + complete_custom_sensor(&replica, &successor, &stale).await, + Err(ControlError::Fenced) + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let recovered = begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned callback") + .expect("due after minimum interval"); + assert_eq!(recovered.cursor.custom.as_deref(), Some("orders/page-3")); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale queued request") + .len(), + test_case.expected_runs + ); + assert_eq!( + evaluations(store) + .await + .iter() + .filter(|fact| fact.request.evaluation_id == stale.evaluation_id + && fact.outcome == SensorEvaluationOutcome::Abandoned) + .count(), + 1 + ); + assert_eq!( + client + .poll(&poll) + .await + .expect("new epoch work") + .expect("new claim") + .request + .request_id, + recovered.evaluation_id, + "retired evaluation must not block or execute before current work" + ); + close_store(replica).await; + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs index 1b4e85322d..af27d0d35e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -1,4 +1,5 @@ use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::sensors::models::CustomSensorOutcome; use crate::store::tests::store_test_types::Backend; pub(crate) struct SensorTestCase { @@ -7,6 +8,12 @@ pub(crate) struct SensorTestCase { pub(crate) expected_runs: usize, } +pub(crate) struct CustomSensorResultCase { + pub(crate) outcome: CustomSensorOutcome, + pub(crate) expected_phase: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + pub(crate) struct SensorExecutionTestCase { pub(crate) description: &'static str, pub(crate) outcome: RunOutcome, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 4b4c372cee..fd3e005f42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -775,12 +775,14 @@ fn parsed_invocation<'py>( let namespace = args.getattr("__dict__")?.cast_into::()?; let command: Option = namespace .get_item("command")? - .map(|value| value.extract()) - .transpose()?; + .map(|value| value.extract::>()) + .transpose()? + .flatten(); let location_command: Option = namespace .get_item("location_command")? - .map(|value| value.extract()) - .transpose()?; + .map(|value| value.extract::>()) + .transpose()? + .flatten(); if command.as_deref() == Some("location") && location_command.as_deref() == Some("runtime") { return within(&host::redirect_stdout_to_stderr(py)?, |_| { execute_parsed_invocation(args, handlers, identity, use_color) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index caab6c25e6..c04d8a5435 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -202,8 +202,8 @@ pub(crate) fn compile_declarations( triggers, sensors, } = compile_sensors( - &input.target, - graph, + py, + (project_dir, input.target.as_str(), graph), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 6a0505dbda..7a5f85c971 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -3,12 +3,14 @@ use std::io::{self, Write}; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; +use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; @@ -22,23 +24,57 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul let Some(line) = line else { break; }; - let generation: i64 = serde_json::from_str(&line) - .map_err(|_| PyValueError::new_err("runtime expects a JSON generation per line"))?; - options.set_item("generation", generation)?; - let arguments = location_arguments(py, Some(options.clone()))?; - let record = match publish_location(py, &arguments) { - Ok(record) => Some(record), - Err(error) => { - write_console( - py, - "stderr", - &format!("Location compilation/publication failed: {error}\n"), - )?; - None + let request: serde_json::Value = serde_json::from_str(&line).map_err(|_| { + PyValueError::new_err( + "runtime expects a JSON generation or sensor evaluation per line", + ) + })?; + let output = if let Some(generation) = request.as_i64() { + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_console( + py, + "stderr", + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + serde_json::to_string(&record) + } else { + let evaluation: SensorEvaluationRequest = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid sensor evaluation") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + if arguments.value("location_id")?.extract::()? + != evaluation.location_id.to_string() + || arguments.value("selected_target")?.extract::()? != evaluation.target + { + return Err(PyValueError::new_err( + "sensor work belongs to another location or target", + )); } + let directory = arguments + .value("project_dir")? + .extract::()?; + let outcome = match evaluate_sensor(py, &directory, &evaluation) { + Ok(outcome) => outcome, + Err(_) => { + write_console( + py, + "stderr", + "Custom sensor evaluation failed. No cursor or run requests committed.\n", + )?; + CustomSensorOutcome::Failed + } + }; + serde_json::to_string(&outcome) }; - let output = serde_json::to_string(&record) - .map_err(|error| PyValueError::new_err(error.to_string()))?; + let output = output.map_err(|error| PyValueError::new_err(error.to_string()))?; py.detach(|| -> io::Result<()> { let mut stdout = io::stdout().lock(); writeln!(stdout, "{output}")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index b7e704b8c0..035588e5e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,3 +4,6 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; + +/// The sensor context is supplied by the host, not a provider. +pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index ad5619410e..cb882f735f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod host; pub(crate) mod model_lookup; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; +pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 453a9f5145..12796bb6fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -63,7 +63,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) struct AuthoredSensor { pub(crate) name: String, pub(crate) description: String, - pub(crate) when: AuthoredSensorCondition, + pub(crate) when: Option, + pub(crate) function: Option>, pub(crate) run: AuthoredJobReference, #[pyo3(from_py_with = authored_setting)] pub(crate) minimum_interval_seconds: Value, @@ -163,7 +164,9 @@ pub(crate) fn collect_declarations<'py>( } pending.push(target); let condition = declaration.getattr("when")?; - if condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION { + if !condition.is_none() + && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION + { let source = condition.getattr("job")?; if !source.is_instance(&job)? && !source.is_instance(&group)? { return Err(PyTypeError::new_err( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs index 5b49bcaa07..26cfa2bf67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -2,8 +2,12 @@ use std::path::Path; -use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sha2::{Digest, Sha256}; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; /// A lockfile binds declared dependencies; without one, bind installed distribution versions. pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { @@ -48,3 +52,57 @@ pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult PyResult { + let tree = ProjectTree::new(project_dir); + let files = rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir && entry.name.ends_with(PYTHON_FILE_SUFFIX) + }) + .map_err(|error| { + pyo3::exceptions::PyValueError::new_err(format!( + "sensor source discovery failed: {error:?}" + )) + })?; + let mut hash = Sha256::new(); + for file in files { + let bytes = std::fs::read(tree.absolute(&file))?; + hash.update(file.len().to_le_bytes()); + hash.update(file.as_bytes()); + hash.update(bytes.len().to_le_bytes()); + hash.update(bytes); + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Include the callable's code even for public in-memory declaration compilation. +pub(crate) fn callback_code_hash(function: &Bound<'_, PyAny>, sources: &str) -> PyResult { + let bytes: Vec = function + .py() + .import("marshal")? + .call_method1("dumps", (portable_code(function.getattr("__code__")?)?,))? + .extract()?; + let mut hash = Sha256::new(); + hash.update(sources.as_bytes()); + hash.update(bytes); + Ok(format!("{:x}", hash.finalize())) +} + +/// Code objects retain absolute import paths, including inside nested functions. +fn portable_code(code: Bound<'_, PyAny>) -> PyResult> { + let py = code.py(); + let code_type = py.import("types")?.getattr("CodeType")?; + let mut constants: Vec> = Vec::new(); + for value in code.getattr("co_consts")?.try_iter()? { + let value = value?; + constants.push(if value.is_instance(&code_type)? { + portable_code(value)? + } else { + value + }); + } + let changes = PyDict::new(py); + changes.set_item("co_filename", "")?; + changes.set_item("co_consts", PyTuple::new(py, constants)?)?; + code.call_method("replace", (), Some(&changes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs new file mode 100644 index 0000000000..e6c05e8861 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -0,0 +1,132 @@ +//! User callbacks and provider lifetimes belong only to the isolated location interpreter. + +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, +}; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; + +pub(crate) fn evaluate_sensor( + py: Python<'_>, + project_dir: &Path, + evaluation: &SensorEvaluationRequest, +) -> PyResult { + let SensorCondition::Custom { + function: name, + code_hash, + } = &evaluation.definition.when + else { + return Err(PyValueError::new_err("custom sensor work required")); + }; + let public = py.import("sqlbuild.orchestration")?; + let sensor_class = public.getattr("Sensor")?; + let mut matches: Vec> = Vec::new(); + for declaration in discover_declarations(py, project_dir)? { + if declaration.is_instance(&sensor_class)? + && declaration.getattr("name")?.extract::()? == *name + { + matches.push(declaration); + } + } + let [sensor] = matches.as_slice() else { + return Err(PyValueError::new_err( + "custom sensor declaration must resolve uniquely", + )); + }; + let targets: Vec = sensor.getattr("targets")?.extract()?; + if !sensor.getattr("when")?.is_none() + || (!targets.is_empty() && !targets.contains(&evaluation.target)) + || sensor + .getattr("run")? + .getattr("name")? + .extract::()? + != evaluation.definition.job_key + { + return Err(PyValueError::new_err("custom sensor definition changed")); + } + let function = sensor.getattr("function")?; + let sources = py.detach(|| orchestration_code_hash(project_dir))?; + if callback_code_hash(&function, &sources)? != *code_hash { + return Err(PyValueError::new_err("custom sensor code changed")); + } + let fields = PyDict::new(py); + fields.set_item("cursor", &evaluation.cursor.custom)?; + fields.set_item("target", &evaluation.target)?; + fields.set_item("evaluation_id", evaluation.evaluation_id.to_string())?; + let context = public.getattr("SensorContext")?.call((), Some(&fields))?; + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + within(&session, |_| { + let arguments = PyDict::new(py); + arguments.set_item("ctx", &context)?; + for usage in provider_usages(&function, &by_name)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + } + let result = function.call((), Some(&arguments))?; + if result.is_instance(&public.getattr("SkipReason")?)? { + let reason: String = result.getattr("reason")?.extract()?; + if reason.trim().is_empty() { + return Err(PyValueError::new_err("SkipReason needs a nonempty reason")); + } + return Ok(CustomSensorOutcome::Skipped { + cursor: context.getattr("cursor")?.extract()?, + reason, + }); + } + let request_class = public.getattr("RunRequest")?; + let mut run_keys: Vec = Vec::new(); + for request in result + .try_iter()? + .take(sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS + 1) + { + let request = request?; + if !request.is_instance(&request_class)? { + return Err(PyValueError::new_err( + "custom sensors return RunRequest records or SkipReason", + )); + } + let key: String = request.getattr("run_key")?.extract()?; + if key.trim().is_empty() { + return Err(PyValueError::new_err("RunRequest needs a nonempty run_key")); + } + run_keys.push(key); + } + if run_keys.len() > sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS { + return Err(PyValueError::new_err( + "custom sensor exceeds the bounded run-request page", + )); + } + Ok(CustomSensorOutcome::Completed { + cursor: context.getattr("cursor")?.extract()?, + run_keys, + }) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 03f6bb4e77..256be90f0c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -26,6 +26,9 @@ use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredSensor, authored_setting, }; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; #[derive(FromPyObject)] struct AuthoredControls { @@ -250,11 +253,16 @@ pub(crate) fn compile_calendars( } pub(crate) fn compile_sensors( - target: &str, - graph: &ProjectGraph, + py: Python<'_>, + (project_dir, target, graph): (&Path, &str, &ProjectGraph), authored: Vec, mut definitions: Definitions, ) -> PyResult { + let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { + orchestration_code_hash(project_dir)? + } else { + String::new() + }; let mut names: BTreeSet = BTreeSet::new(); for sensor in authored { if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { @@ -266,18 +274,31 @@ pub(crate) fn compile_sensors( if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { continue; } - let mut condition = json!({"kind": sensor.when.kind, "kinds": sensor.when.kinds}); - if let Some(job) = sensor.when.job { - condition["job_key"] = json!(job.name); - } - if let Some(select) = sensor.when.select { - let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; - } - if let Some(selectors) = sensor.when.executed_successfully { - condition["executed_successfully"] = - sensor_nodes(graph, &selectors, "executed_successfully")?; - } + let condition = match (sensor.when, sensor.function) { + (Some(when), None) => { + let mut condition = json!({"kind": when.kind, "kinds": when.kinds}); + if let Some(job) = when.job { + condition["job_key"] = json!(job.name); + } + if let Some(select) = when.select { + let selectors: Vec = + select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; + } + if let Some(selectors) = when.executed_successfully { + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; + } + condition + } + (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, + "code_hash": callback_code_hash(function.bind(py), &sources)?}), + _ => { + return Err(PyValueError::new_err( + "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", + )); + } + }; let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, "job_key": sensor.run.name, diff --git a/fensu.toml b/fensu.toml index 8c5126e9c4..16d15444db 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2655,6 +2655,7 @@ module_layers = [ "model_config=front", "native_host=infra", "native_host/model_lookup=project", + "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", "native_threads=infra", "output_capture=output", @@ -2750,3 +2751,8 @@ rule = "FFS120" path = "src/sqlbuild/executor/scheduling/classes/recovering_connection_pool.py" symbols = ["RecoveringConnectionPool.put"] reason = "The override keeps queue.Queue.put's positional protocol, which every pool consumer calls." + +[[targets.python.rule_exceptions]] +rule = "FFS201" +path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" +reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." diff --git a/pyproject.toml b/pyproject.toml index c29137b916..daf4fa4c8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.1", + "fensu>=0.28.3", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/python-boundary.toml b/python-boundary.toml index 159acb56e3..5829d6322f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -18,10 +18,15 @@ rule = 3 note = "Public orchestration facade: re-exports only." [[stays]] -path = "orchestration_declarations/models.py" +path = "orchestration_declarations/models/authoring.py" rule = 3 note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." +[[stays]] +path = "orchestration_declarations/models/sensor_context.py" +rule = 3 +note = "Only the mutable SensorContext authoring declaration required by ctx.cursor assignment; no Python runtime allowance. Native fenced completion validates and commits proposed progress." + [[stays]] path = "orchestration_declarations/types.py" rule = 3 diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 3532cf5f7d..568ce36709 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,7 +1,7 @@ """Public authoring and snapshot compilation API for orchestration declarations.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models import ( +from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule, CalendarWindow, Job, @@ -9,17 +9,24 @@ NodeEventCondition, PlatformEventCondition, RunEventCondition, + RunRequest, Schedule, Sensor, + SkipReason, ) -from sqlbuild.orchestration_declarations.models import CalendarSchedule as calendar_schedule -from sqlbuild.orchestration_declarations.models import Job as job -from sqlbuild.orchestration_declarations.models import JobGroup as job_group -from sqlbuild.orchestration_declarations.models import NodeEventCondition as node_event -from sqlbuild.orchestration_declarations.models import PlatformEventCondition as platform_event -from sqlbuild.orchestration_declarations.models import RunEventCondition as run_event -from sqlbuild.orchestration_declarations.models import Schedule as schedule -from sqlbuild.orchestration_declarations.models import Sensor as sensor +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule as calendar_schedule, +) +from sqlbuild.orchestration_declarations.models.authoring import Job as job +from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group +from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event +from sqlbuild.orchestration_declarations.models.authoring import ( + PlatformEventCondition as platform_event, +) +from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event +from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule +from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor +from sqlbuild.orchestration_declarations.models.sensor_context import SensorContext from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind __all__ = ( @@ -33,8 +40,11 @@ "PlatformEventKind", "RunEventCondition", "RunEventKind", + "RunRequest", "Schedule", "Sensor", + "SensorContext", + "SkipReason", "calendar_schedule", "compile_snapshot", "job", diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index cc7f6c83f2..0f9fc8b052 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -7,7 +7,12 @@ from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models import CalendarSchedule, Job, JobGroup, Sensor +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule, + Job, + JobGroup, + Sensor, +) def compile_snapshot( diff --git a/src/sqlbuild/orchestration_declarations/models/__init__.py b/src/sqlbuild/orchestration_declarations/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/orchestration_declarations/models.py b/src/sqlbuild/orchestration_declarations/models/authoring.py similarity index 87% rename from src/sqlbuild/orchestration_declarations/models.py rename to src/sqlbuild/orchestration_declarations/models/authoring.py index 8f4a44a9b8..fd188208ed 100644 --- a/src/sqlbuild/orchestration_declarations/models.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -115,8 +115,30 @@ class Sensor: name: str description: str = field(kw_only=True) - when: PlatformEventCondition | RunEventCondition | NodeEventCondition = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition | NodeEventCondition | None = field( + default=None, kw_only=True + ) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( + default=None, repr=False, kw_only=True + ) + + def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class RunRequest: + """A sensor-local key deduplicates requests across evaluations and retries.""" + + run_key: str + + +@dataclass(frozen=True) +class SkipReason: + """Complete an evaluation without starting a run.""" + + reason: str diff --git a/src/sqlbuild/orchestration_declarations/models/sensor_context.py b/src/sqlbuild/orchestration_declarations/models/sensor_context.py new file mode 100644 index 0000000000..648ce981db --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/sensor_context.py @@ -0,0 +1,12 @@ +"""Invocation-local context; native fenced completion alone commits a proposed cursor.""" + +from dataclasses import dataclass + + +@dataclass(kw_only=True) +class SensorContext: + """Cursor assignment proposes progress; failure or timeout retains the committed cursor.""" + + cursor: str | None + target: str + evaluation_id: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index f1f32da930..8f5aca9f03 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -47,11 +47,13 @@ def write(self, text: str) -> int: @contextmanager def location_process( - arguments: list[str], log: Path, publications: Path + arguments: list[str], log: Path, publications: Path, *, env: dict[str, str] | None = None ) -> Iterator[subprocess.Popen[bytes]]: """Own and reap the supervisor even when an assertion interrupts a reload.""" with log.open("ab") as stderr, publications.open("ab") as stdout: - process: subprocess.Popen[bytes] = subprocess.Popen(arguments, stderr=stderr, stdout=stdout) + process: subprocess.Popen[bytes] = subprocess.Popen( + arguments, stderr=stderr, stdout=stdout, env=env + ) try: yield process finally: @@ -563,3 +565,28 @@ def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any ), ) return ClaimedWorker(run_id=run_id, claim_id=claim_id, bearer=bearer) + + +def wait_for_sensor_phase( + store: Path, phase: str, *, empty_runs: bool = False, timeout: float = 45 +) -> dict[str, Any]: + """Wait for real scheduler facts, optionally requiring a completed skip.""" + deadline: float = time.monotonic() + timeout + row: tuple[str] | None = None + while row is None: + assert time.monotonic() < deadline, f"Sensor did not record {phase}" + with sqlite3.connect(store) as control: + row = control.execute( + "SELECT fact_json FROM sqb_sensor_evaluation_events WHERE phase=? " + "AND (? = FALSE OR fact_json->'outcome'->'run_ids'='[]') " + "ORDER BY storage_id DESC LIMIT 1", + (phase, empty_runs), + ).fetchone() + time.sleep(0.05) + return json.loads(row[0]) + + +def sensor_calls(path: Path, fields: dict[str, Any]) -> list[dict[str, Any]]: + """Select recorded callback invocations without inferring execution from declarations.""" + records: list[dict[str, Any]] = [json.loads(line) for line in path.read_text().splitlines()] + return list(filter(lambda record: fields.items() <= record.items(), records)) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py new file mode 100644 index 0000000000..1c6fbcdc4c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -0,0 +1,160 @@ +"""A real location callback times out, restarts and queues a real DuckDB worker.""" + +import ast +import json +import os +import sqlite3 +import sys +from pathlib import Path +from typing import Any, cast + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + scheduled_worker, + sensor_calls, + wait_for_generation, + wait_for_run, + wait_for_sensor_phase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "polling fulfillment sensor", + "SELECT 83 AS amount", + "SELECT amount FROM orders", + "success", + [(83,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + "providers/fulfillment.py": ( + "import json, os, time\nfrom pathlib import Path\n" + "from sqlbuild.orchestration import RunRequest, SkipReason\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Synthetic fulfillment readiness."""\n' + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_SENSOR_CALLS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def poll(self, ctx):\n" + " mode = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())['mode']\n" + " self.record({'phase': 'evaluate', 'cursor': ctx.cursor, 'evaluation_id': ctx.evaluation_id, 'target': ctx.target, 'mode': mode})\n" + " if mode == 'failed':\n" + " ctx.cursor = 'uncommitted'\n" + " raise ValueError('synthetic source unavailable')\n" + " if mode == 'timeout':\n time.sleep(90)\n" + " if ctx.cursor:\n return SkipReason('No new orders')\n" + " ctx.target = 'not-an-authority-input'\n" + " ctx.evaluation_id = 'not-an-authority-input'\n" + " ctx.cursor = 'orders/page-1'\n" + " return [RunRequest(run_key='orders-1'), RunRequest(run_key='orders-1')]\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a_fresh_process_retries_without_advancing_its_cursor( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + context_module: ast.Module = ast.parse( + Path("src/sqlbuild/orchestration_declarations/models/sensor_context.py").read_text() + ) + assert [type(node).__name__ for node in context_module.body] == [ + "Expr", + "ImportFrom", + "ClassDef", + ] + declaration: ast.ClassDef = cast(ast.ClassDef, context_module.body[-1]) + assert declaration.name == "SensorContext" + assert [type(node).__name__ for node in declaration.body] == [ + "Expr", + "AnnAssign", + "AnnAssign", + "AnnAssign", + ] + data: Path = tmp_path / "fulfillment.json" + calls: Path = tmp_path / "sensor-calls.jsonl" + data.write_text(json.dumps({"mode": "failed"})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_SENSOR_CALLS", str(calls)) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "publications.jsonl", + ) as location: + _ = wait_for_generation(worker.api, "development", 2, location, log) + failed: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "failed") + assert failed["request"]["cursor"] == {"event_storage_id": 0} + data.write_text(json.dumps({"mode": "timeout"})) + timed_out: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "timed_out") + assert timed_out["request"]["cursor"] == failed["request"]["cursor"] + timed_call: dict[str, Any] = sensor_calls( + calls, {"evaluation_id": timed_out["request"]["evaluation_id"]} + )[0] + with pytest.raises(ProcessLookupError): + os.kill(timed_call["pid"], 0) + data.write_text(json.dumps({"mode": "ready"})) + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, log.read_text() + # The next successful evaluation skips but retains its committed cursor. + skipped: dict[str, Any] = wait_for_sensor_phase( + worker.control_store, "completed", empty_runs=True + ) + assert skipped["outcome"]["cursor"]["custom"] == "orders/page-1" + evaluated: list[dict[str, Any]] = sensor_calls(calls, {"phase": "evaluate"}) + assert {record["target"] for record in evaluated} == {"development"} + assert len({record["pid"] for record in evaluated}) == len(evaluated) + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "failed"}) + } == {None} + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "timeout"}) + } == {None} + assert location.poll() is None and worker.process.poll() is None + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT run_key,run_id FROM sqb_run_requests" + ).fetchall() == [("orders-1", destination["run_id"])] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert ( + warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + ) + assert not (worker.project / ".sqlbuild/control.sqlite3").exists() + assert worker.api.bearer not in log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index abc6cdea92..b8e7303b67 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -77,7 +77,8 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ with snapshot_api(tmp_path / "control") as api: monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) # A presentation-only width fallback must not bypass the native-only supervisor. - monkeypatch.setenv("COLUMNS", "999999999999999999999999999999") + # Keep the adversarial width out of pytest's own terminal reporter. + environment: dict[str, str] = os.environ | {"COLUMNS": "999999999999999999999999999999"} arguments: list[str] = [ str(Path(sys.executable).with_name("sqb")), "--project-dir", @@ -93,7 +94,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ] log: Path = project / "location.log" publications: Path = project / "publications.jsonl" - with location_process(arguments, log, publications) as process: + with location_process(arguments, log, publications, env=environment) as process: first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) first_interpreter: int = int(compiler_pid.read_text()) heartbeat: dict[str, Any] = wait_for_location_health( @@ -182,7 +183,7 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) process.terminate() assert process.wait(timeout=5) == test_case.expected_stop_code - with location_process(arguments, log, publications) as process: + with location_process(arguments, log, publications, env=environment) as process: restarted: dict[str, Any] = wait_for_generation( api, test_case.target, test_case.expected_generation, process, log ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index ded89951dd..6f6cc7a182 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -123,6 +123,16 @@ }, frozenset({"model", "source"}), ), + SnapshotCatalogTestCase( + "DuckShop polling callback with nested code is relocatable", + TYPE_PROOF_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + }, + frozenset({"model"}), + ), SnapshotCatalogTestCase( "DuckShop inherited seed and function namespaces", TYPE_PROOF_FILES @@ -801,6 +811,32 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=TypeError, expected_message="group members must be Job objects", ), + SnapshotDiscoveryRejectionTestCase( + description="a polling sensor needs an actual callback", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "poll = sensor('poll', description='Poll orders.', run=orders)\n" + ), + }, + expected_error=ValueError, + expected_message="either a built-in condition or a decorated polling callback", + ), + SnapshotDiscoveryRejectionTestCase( + description="callbacks with event conditions remain explicitly unsupported", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "@sensor('poll', description='Poll orders.', run=orders, " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" + "def poll(ctx):\n return []\n" + ), + }, + expected_error=ValueError, + expected_message="callbacks with when are not supported yet", + ), SnapshotDiscoveryRejectionTestCase( description="explicit group cycles cannot publish", files={ diff --git a/uv.lock b/uv.lock index 17225a2936..d0a6aa78ca 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.1" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ab/c7/baf72530c3a67a6d26957658475d32e11adbeb2dbb452425fd5d4af1f575/fensu-0.28.1.tar.gz", hash = "sha256:b81119f0301580367799b77312d5fd82f3ce5a14613b5b709537bca4c070b912", size = 539035, upload-time = "2026-10-31T19:33:41.927Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/3d/d5b4fb0ad09426dae015af2817ba8416d2ea974a0c4a6ebea63112f90b1b/fensu-0.28.3.tar.gz", hash = "sha256:a7fc87cb60eaef964b4e980d19bd6de9c95cecc75c019d8b0058e02b6751f4b5", size = 539031, upload-time = "2026-11-01T09:41:23.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/02/8288dc1ecb853941750d25c2b1effc6fe68d9eeeb188f3d289823731cf69/fensu-0.28.1-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bf1d49d2c800e7c8acdba031c12e6ea1a38544f45c0b75fbe8c35369b63ee98b", size = 4154268, upload-time = "2026-10-31T19:33:24.239Z" }, - { url = "https://files.pythonhosted.org/packages/4c/70/f21686ef9b89f70c84ea3a94dcdbda47839263496c36aa962c15c2eb093e/fensu-0.28.1-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:7e7564b2c4ffb9a031659b3bd82368be70deb97c3f4f620ad05ddfb5f8086e9c", size = 4004303, upload-time = "2026-10-31T19:33:25.807Z" }, - { url = "https://files.pythonhosted.org/packages/3c/50/558ac8fadf943aad5a8a5ba500a3f63e0f5245fd1f66480b6cd8590ce599/fensu-0.28.1-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70da92dad5e5dfbe927bfe870893fe8dc612f28e72780de32014ca42703ba7db", size = 4230255, upload-time = "2026-10-31T19:33:27.649Z" }, - { url = "https://files.pythonhosted.org/packages/5b/5f/b7659a2d6489293a6c16cafc5f8c500b5d5de22536767f468147bed813e3/fensu-0.28.1-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:207a911faee0e14f40005060281ea624b6360f7f35cf6440a02dd0870c4f0bfc", size = 4307330, upload-time = "2026-10-31T19:33:29.361Z" }, - { url = "https://files.pythonhosted.org/packages/69/a6/aa98ad24bb27e3c829c9675c2b753f5d793fb5f546e8b266ade494d6e044/fensu-0.28.1-cp312-abi3-win_amd64.whl", hash = "sha256:88089637c5a52a2e074dd7b29fb76d4e35d1d9d28e8ef4398be30c3bed7d8071", size = 3977891, upload-time = "2026-10-31T19:33:30.84Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a7/9493fbcf3158ae009651d19c7289855e8ef571dfaf5c7c77154a8a6a4a21/fensu-0.28.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e3896f173d2bb676fe04defdd4ebdf4db38aee3ee6e57d7a5a297df32d7c5c", size = 4154274, upload-time = "2026-11-01T09:41:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/20fdb7147e268c5dbd9eae0fc9dd99641e416f9e75e8fd067ac089f6db78/fensu-0.28.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d74db7466b2ddd533413d2daf4c704aa4acbcae61e2a0edba2564a4910666707", size = 4004309, upload-time = "2026-11-01T09:41:05.719Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4c/40b01d8e46ff3464eb503529e00ad23eddd2ffbd605282618672d0aff45f/fensu-0.28.3-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da5843aa40c7d6398a95b340b3c60d4360b5feddac34960663ee84aec09df8ef", size = 4230261, upload-time = "2026-11-01T09:41:07.481Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/243cccd83531c3c86c660b993e77a7014c65d1d3f20c07e45b067dba81cf/fensu-0.28.3-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52e1623e614f101c5ada2eda59548ff441b7efe4228813e69bc0f0ef9b3fbca6", size = 4307334, upload-time = "2026-11-01T09:41:09.938Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/38dbb50d5079a1a5fcaf62bcdd582c12a53609c575a747cb2461e83ddfcf/fensu-0.28.3-cp312-abi3-win_amd64.whl", hash = "sha256:a39c407725ea4d92123414bdbe9f80d5db0b7a412af49082a87378203c9ecb04", size = 3977904, upload-time = "2026-11-01T09:41:11.672Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.1" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/da/a4ef081ab0ec23cb719dfbce2747c11ee3832a2dda2aaef1f7722548d1e6/fensu_cli-0.28.1.tar.gz", hash = "sha256:4857b2ef850d35bae19462cd6e26fdc0b33c00d97f13f20bb5b00b5c879ef2e9", size = 765839, upload-time = "2026-10-31T19:33:43.051Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/17/3d0953c33387caa0a6becc9bb02210d4a020eaaf73ef07bdc0c8f8952562/fensu_cli-0.28.3.tar.gz", hash = "sha256:3ba38bedc88a9e6db8563f638aa6ba8649a3e5750de176e22a7b35561d14bd9e", size = 766571, upload-time = "2026-11-01T09:41:25.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/03/1c5a56b3d233360d1fb699ff7a92cee005faf8f8fda8c92a7688cc0be167/fensu_cli-0.28.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:67b0bc97cdaeb08e633e68336b1d82b3e598d615b54b9b6f840772929cf6929e", size = 7975688, upload-time = "2026-10-31T19:33:32.569Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a7/d90a388b17a89d9e34238e16e03ce306331c212d215f02ff4c43d6d93189/fensu_cli-0.28.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6a9d97525aea16e079b773b7141a1167bd799bc07ff8bfc720b8f311c91d20b", size = 7492140, upload-time = "2026-10-31T19:33:34.634Z" }, - { url = "https://files.pythonhosted.org/packages/62/f3/4a5247e312d4725ead4ab37d0076371b731a8858c5819cfcceb1d6ce0b69/fensu_cli-0.28.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b8fd61d84ffd2be3c023335e374cabc72eac74dfe5cd90103bde52eba6c40da5", size = 7860157, upload-time = "2026-10-31T19:33:36.407Z" }, - { url = "https://files.pythonhosted.org/packages/d4/71/a954282a9f3f372fe77c942ff6acc8cc2f797b7da82b3b40a6c69715b668/fensu_cli-0.28.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1115a28579b917d6d8c0010e99488318e4cd264236642b5081bb79e8378bdc15", size = 8323299, upload-time = "2026-10-31T19:33:38.357Z" }, - { url = "https://files.pythonhosted.org/packages/35/cb/aa860d0afcb17a7931ab14dcab061d234143caef12e2791d17f27d0816c3/fensu_cli-0.28.1-py3-none-win_amd64.whl", hash = "sha256:4294965dcf4c317b66e4e489c9b9b38e8f8d82000c59bf9f3275fe887a6b45d8", size = 7705331, upload-time = "2026-10-31T19:33:40.314Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bb/09753cc0584c21af1d76eca7c9a6ea5e2d876d71c079c5505dda93742f3d/fensu_cli-0.28.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5179b032074f4ff5097950c0c0b063fd872cffcd23a4bf8a89c71ab0d3d8074b", size = 7963143, upload-time = "2026-11-01T09:41:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/47/59/663fb61d56172bfb053375e8623a23db389b746052fb00dd087cf6c1a76b/fensu_cli-0.28.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41077b9ec6e2eb46405af809d1010b26519c8928d4264137ca4c6d47ce81d2e1", size = 7479791, upload-time = "2026-11-01T09:41:16.253Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7c/1a03bd25bc2a231ca03478ba3dfd8f21c909eae564c3df751a8982222beb/fensu_cli-0.28.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ac929d598046846c18eae62c8b75a9a49e9b6bbcadb46bd413a5da96ef6453e2", size = 7854377, upload-time = "2026-11-01T09:41:18.109Z" }, + { url = "https://files.pythonhosted.org/packages/17/5e/8395e4bd4e9e4d282a0949e42e721101127fb30f45cb71d6e5af05329051/fensu_cli-0.28.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ce453889008381184a416acaf274ffa4a46c7cb96ad4397de264816bb10626a", size = 8316670, upload-time = "2026-11-01T09:41:20.045Z" }, + { url = "https://files.pythonhosted.org/packages/89/40/6e3831cd39b16f163916b36c03010754c1cfb58583fd737517bd544751d5/fensu_cli-0.28.3-py3-none-win_amd64.whl", hash = "sha256:66ec08187a6dcc43392244ba4b7349ee29e8da63fbe7110114468f07a000d577", size = 7710605, upload-time = "2026-11-01T09:41:22.073Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.1" }, + { name = "fensu", specifier = ">=0.28.3" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From ed9783d49c936f70486066fdd34232ca088f1a0f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:59:27 +0000 Subject: [PATCH 055/138] feat: persist warehouse observation identities and read history Add nullable deterministic event IDs to fingerprint and source-freshness writers without rewriting predecessor rows or changing latest-state reads. Preserve source identities under location, target and state-table scoping; use fixed legacy fields only when the source ID is absent. Keep warehouse history reads genuinely read-only and bounded. Fence atomic ingestion pages to the current location session and generation, and preserve transient retries during populated fingerprint upgrades. Document the identity contract separately from unfinished automatic synchronization. --- Cargo.lock | 1 + .../orchestration/legacy_warehouse_state.sql | 22 ++ crates/README.md | 25 ++ .../src/fingerprint_storage/constants.rs | 3 +- .../src/fingerprint_storage/main/_event_id.rs | 20 ++ .../main/create_table_sql.rs | 10 +- .../fingerprint_storage/main/insert_sql.rs | 7 +- .../src/fingerprint_storage/main/mod.rs | 1 + .../fingerprint_storage/tests/test_storage.rs | 5 +- crates/sqlbuild-orchestration/Cargo.toml | 1 + .../src/api/_helpers/mod.rs | 1 + .../src/api/_helpers/warehouse_metadata.rs | 30 ++ .../src/api/main/serve_api.rs | 6 +- .../sqlbuild-orchestration/src/api/models.rs | 5 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/mod.rs | 1 - .../src/platform/_helpers/work_client.rs | 21 ++ .../src/platform/main/claim_location_work.rs | 2 +- .../platform/main/complete_location_work.rs | 2 +- .../src/platform/main/mod.rs | 1 + .../require_instance.rs} | 2 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/warehouse_freshness.sql | 10 + .../src/warehouse_metadata/constants.rs | 3 + .../main/ingest_source_freshness.rs | 103 ++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/mod.rs | 5 + .../src/warehouse_metadata/models.rs | 56 ++++ .../tests/api/test_contracts.rs | 2 + .../tests/api/test_location_work.rs | 2 + .../tests/api/test_types.rs | 6 + .../tests/api/test_warehouse_metadata.rs | 306 ++++++++++++++++++ .../tests/sensors/test_runs.rs | 3 + .../tests/snapshots/helpers.rs | 3 +- .../_helpers/exported_constants/constants.rs | 9 +- .../_helpers/fingerprints/storage_io.rs | 73 +++-- .../_helpers/freshness/publication.rs | 5 + .../bindings/_helpers/freshness/state_io.rs | 34 +- .../_helpers/location_command/runtime.rs | 20 ++ .../src/bindings/_helpers/native_host/mod.rs | 2 + .../native_host/state_column_evolution.rs | 85 +++++ .../native_host/warehouse_metadata.rs | 205 ++++++++++++ .../bindings/_helpers/records/constants.rs | 2 + .../bindings/_helpers/state_rows/freshness.rs | 38 ++- .../_helpers/state_rows/state_tables.rs | 10 +- .../sqlbuild-state/src/freshness/constants.rs | 5 +- .../src/freshness/main/event_id.rs | 22 ++ .../sqlbuild-state/src/freshness/main/mod.rs | 2 + .../src/freshness/main/read_history_sql.rs | 112 +++++++ crates/sqlbuild-state/src/freshness/models.rs | 15 + fensu.toml | 1 + src/sqlbuild/_native.pyi | 9 + .../contract/classes/duckdb_backed_adapter.py | 4 +- .../sqlserver/classes/sqlserver_adapter.py | 4 +- .../executor/clone/_helpers/fingerprinting.py | 2 + .../executor/diff/_helpers/query_artifacts.py | 2 + .../executor/functions/main/_execute.py | 2 + .../python_nodes/_helpers/fingerprinting.py | 2 + .../executor/review/_helpers/marker.py | 2 + .../run/_helpers/reuse/fingerprinting.py | 2 + .../executor/seed/_helpers/fingerprinting.py | 2 + .../publication/_test_types.py | 11 + .../compiled_project/publication/helpers.py | 56 +++- .../publication/test_warehouse_metadata.py | 293 +++++++++++++++++ 65 files changed, 1643 insertions(+), 59 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql create mode 100644 crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs rename crates/sqlbuild-orchestration/src/platform/{_helpers/location_work.rs => main/require_instance.rs} (90%) create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/event_id.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/read_history_sql.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py diff --git a/Cargo.lock b/Cargo.lock index 8a5982d252..37cab0cc6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2597,6 +2597,7 @@ dependencies = [ "serde", "serde_json", "sqlbuild-recall", + "sqlbuild-state", "sqlx-core", "sqlx-postgres", "subtle", diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql new file mode 100644 index 0000000000..b135ecff51 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql @@ -0,0 +1,22 @@ +-- Populated predecessor schemas for additive event-ID and read-only history-sync checks. +CREATE TABLE main._sqlbuild_fingerprints ( + node_type VARCHAR NOT NULL, node_name VARCHAR NOT NULL, + target_database VARCHAR, target_schema VARCHAR, target_name VARCHAR, + run_id VARCHAR NOT NULL, definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, + schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, + metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_fingerprints VALUES ( + 'model', 'orders', NULL, 'main', 'orders', 'legacy-run', 'legacy-definition', + 'legacy-version', 'legacy-schema', 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2026-01-01 10:00:00' +); +CREATE TABLE main._sqlbuild_source_freshness ( + source_name VARCHAR NOT NULL, target_database VARCHAR, target_schema VARCHAR, + target_name VARCHAR, run_id VARCHAR NOT NULL, strategy VARCHAR NOT NULL, + value_kind VARCHAR NOT NULL, data_version VARCHAR, data_version_hash VARCHAR NOT NULL, + observed_at TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_source_freshness VALUES ( + 'raw.orders', NULL, 'main', 'orders', 'legacy-run', 'max_column', 'integer', + '7', 'legacy-value', '2026-01-01 10:00:00' +); diff --git a/crates/README.md b/crates/README.md index 37c7b39166..41952bc041 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,31 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Warehouse event identity + +Fingerprint and source-freshness writers persist deterministic `event_id` values in nullable text +columns. Existing tables evolve additively; no predecessor rows are rewritten. Latest-state readers +retain their existing projections. IDs are lowercase SHA-256 hex over compact UTF-8 JSON arrays, +whose fixed namespace is independent of the package version: + +| Table | Array fields, in order | +|---|---| +| `_sqlbuild_fingerprints` | `"sqlbuild.fingerprint.event.v1"`, node kind, node name, target database, target schema, target name, run ID, version hash | +| `_sqlbuild_source_freshness` | `"sqlbuild.source-freshness.event.v1"`, source name, target database, target schema, target name, run ID, observation time, strategy, value kind, value hash | + +Nullable identity fields remain JSON null, distinct from empty strings. Freshness observation time +is normalized to UTC ISO text with six fractional digits and a `+00:00` offset. Retrying the same +observation retains its ID; another run or observation is distinct. Definitions, display values, +and future storage columns do not alter these identities. + +The location history reader is read-only, including local database connections: it neither creates +databases nor evolves tables. Source-freshness ingestion preserves warehouse IDs, scoped by location, +target and physical state table. Only missing or NULL IDs use the fixed legacy observation tuple, +never a hash of the evolving full row. Bounded pages commit atomically under the current location +session and snapshot generation, and repeated pages are idempotent. These are read/ingestion +primitives; automatic polling, watermark/overlap management and deletion reconciliation are separate +location-sync responsibilities, not implemented by the page reader. + ## Working on the crates - Tests live next to the module they cover; run them for one crate with diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs index 791dcc6822..59e8a2328c 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -12,8 +12,9 @@ pub const COLUMN_SCHEMA_FINGERPRINT: &str = "schema_fingerprint"; pub const COLUMN_DEFINITION_B64: &str = "definition_b64"; pub const COLUMN_METADATA_JSON_B64: &str = "metadata_json_b64"; pub const COLUMN_TIMESTAMP: &str = "ts"; +pub const COLUMN_EVENT_ID: &str = "event_id"; -/// Every stored column, in table order. +/// Legacy columns used by latest-row readers, including readers of tables without an event ID. pub const FINGERPRINT_COLUMNS: [&str; 12] = [ COLUMN_NODE_TYPE, COLUMN_NODE_NAME, diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs new file mode 100644 index 0000000000..6ef1ccee43 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs @@ -0,0 +1,20 @@ +//! Fixed writer identity excludes timestamps, definitions and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::fingerprint_storage::models::StoredFingerprint; + +/// Stable SHA-256 of the versioned JSON tuple named by the fingerprint storage contract. +pub(super) fn event_id(row: &StoredFingerprint) -> String { + let identity = serde_json::json!([ + "sqlbuild.fingerprint.event.v1", + row.node_type, + row.node_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs index 7c0a00587e..db3a126811 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs @@ -1,9 +1,10 @@ //! Fingerprint table DDL. use crate::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, }; /// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, with the adapter's column types. @@ -21,6 +22,7 @@ pub fn create_table_sql(qualified_name: &str, string_type: &str, timestamp_type: {COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, \ {COLUMN_DEFINITION_B64} {string_type} NOT NULL, \ {COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, \ - {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL)" + {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL, \ + {COLUMN_EVENT_ID} {string_type})" ) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs index 25e54bf696..dbd785c797 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs @@ -3,7 +3,9 @@ use crate::fingerprint_storage::_helpers::base64::encode; use crate::fingerprint_storage::_helpers::literals::{optional_string_literal, string_literal}; use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::constants::COLUMN_EVENT_ID; use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::main::_event_id::event_id; use crate::fingerprint_storage::models::StoredFingerprint; /// One multi-row `INSERT` for a non-empty batch. @@ -16,14 +18,14 @@ pub fn insert_sql( } let rows: Vec = fingerprints.iter().map(row_values).collect(); Ok(format!( - "INSERT INTO {qualified_name} ({}) VALUES {}", + "INSERT INTO {qualified_name} ({}, {COLUMN_EVENT_ID}) VALUES {}", column_list(), rows.join(", ") )) } fn row_values(fingerprint: &StoredFingerprint) -> String { - let values: [String; 12] = [ + let values: [String; 13] = [ string_literal(&fingerprint.node_type), string_literal(&fingerprint.node_name), optional_string_literal(fingerprint.target_database.as_deref()), @@ -36,6 +38,7 @@ fn row_values(fingerprint: &StoredFingerprint) -> String { format!("'{}'", encode(fingerprint.definition.as_bytes())), format!("'{}'", encode(fingerprint.metadata_json.as_bytes())), format!("'{}'", fingerprint.ts), + string_literal(&event_id(fingerprint)), ]; format!("({})", values.join(", ")) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs index 54c1f22e6f..ab782386e7 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs @@ -1,3 +1,4 @@ +mod _event_id; pub mod create_table_sql; pub mod decode_storage; pub mod insert_batch_lengths; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs index 59c72c1dd6..5595b45108 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs @@ -142,8 +142,9 @@ fn given_rows_when_inserting_then_literals_are_escaped_and_definitions_encoded() expected_sql: Ok( "INSERT INTO main.t (node_type, node_name, target_database, target_schema, \ target_name, run_id, definition_hash, version_hash, schema_fingerprint, \ - definition_b64, metadata_json_b64, ts) VALUES ('model', 'o''rders', NULL, \ - 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00')", + definition_b64, metadata_json_b64, ts, event_id) VALUES ('model', 'o''rders', NULL, \ + 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00', \ + 'b9397954c803e7f99f6a233c2985be772c7ddf2c6f234197d77bc227ae175cbd')", ), }, InsertTestCase { diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 4e31942af6..24f97c2b34 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -20,6 +20,7 @@ semver.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-recall.workspace = true +sqlbuild-state.workspace = true sqlx-core.workspace = true sqlx-postgres.workspace = true subtle.workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 996c7546bb..94872aa28d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod run_reads; pub(crate) mod runs; pub(crate) mod snapshots; pub(crate) mod trigger_reads; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs new file mode 100644 index 0000000000..5ddb1f2001 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs @@ -0,0 +1,30 @@ +//! An authenticated location can mirror only its own registered reader's target. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: ingest_source_freshness(&store, (location_id, &target), &batch).await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 37918480f4..dd000306a9 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -6,7 +6,7 @@ use axum::{Json, Router, middleware}; use crate::api::_helpers::authentication::authenticate; use crate::api::_helpers::{ - location_health, location_work, run_reads, runs, snapshots, trigger_reads, + location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; @@ -33,6 +33,10 @@ pub async fn serve_api( "/api/locations/{location_id}/health", get(location_health::recent).post(location_health::report), ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + post(warehouse_metadata::report), + ) .route( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 06471e75a9..9668db92c1 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::api::_helpers::{ - location_health, location_work, run_reads, runs, snapshots, trigger_reads, + location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] @@ -27,7 +27,8 @@ use crate::api::_helpers::{ run_reads::recall, trigger_reads::current, trigger_reads::events, - trigger_reads::ticks + trigger_reads::ticks, + warehouse_metadata::report ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index cc435f1701..3f1421053e 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -11,4 +11,5 @@ pub mod scheduler; pub mod sensors; pub mod snapshots; pub mod store; +pub mod warehouse_metadata; pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index 574aaa4680..fddc0b0e5a 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod liveness; pub(crate) mod location_heartbeat; -pub(crate) mod location_work; pub(crate) mod observations; pub(crate) mod watchdog; pub(crate) mod webhook_delivery; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index fe1e104915..5c1351ac68 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -8,8 +8,29 @@ use crate::api::models::MutationResponse; use crate::platform::models::{ LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, }; +use crate::warehouse_metadata::models::SourceFreshnessBatch; impl LocationWorkClient { + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. + pub async fn sync_source_freshness( + &self, + target: &str, + batch: &SourceFreshnessBatch, + ) -> Result { + let endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "targets", + target, + "metadata", + "source-freshness", + ])?; + let response: MutationResponse = + client_response(self.connection.client.post(endpoint).json(batch)).await?; + Ok(response.applied) + } + pub async fn poll( &self, poll: &LocationWorkPoll, diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs index 7a36796c37..8d071b1626 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -3,8 +3,8 @@ use chrono::Duration; use uuid::Uuid; -use crate::platform::_helpers::location_work::require_instance; use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index a044885fe4..cc10fab99c 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -3,8 +3,8 @@ use serde_json::json; use uuid::Uuid; -use crate::platform::_helpers::location_work::require_instance; use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; use crate::sensors::constants::MAX_SENSOR_RUN_REQUESTS; use crate::sensors::models::CustomSensorOutcome; diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index a782c64402..8124f49518 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -9,5 +9,6 @@ pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; +pub(crate) mod require_instance; pub mod run_platform_sink; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs similarity index 90% rename from crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs rename to crates/sqlbuild-orchestration/src/platform/main/require_instance.rs index 8e2d429a5a..9d64b6068f 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs @@ -1,4 +1,4 @@ -//! Request retries and completion share the same immutable result reader and instance scope. +//! Named work and warehouse observations share the registered location/target boundary. use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index b2c9b5fbf4..6199e096da 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -14,6 +14,7 @@ pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); +pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshness.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 6190e2798a..f2237bd093 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, - SCHEMA, SENSORS, render, verify_migrations, + SCHEMA, SENSORS, WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -36,6 +36,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("run_sensors", RUN_SENSORS), ("executed_nodes", EXECUTED_NODES), ("node_sensors", NODE_SENSORS), + ("warehouse_freshness", WAREHOUSE_FRESHNESS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql b/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql new file mode 100644 index 0000000000..5cdd1500e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql @@ -0,0 +1,10 @@ +CREATE TABLE sqb_warehouse_freshness ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs new file mode 100644 index 0000000000..05ee34c05a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -0,0 +1,3 @@ +//! Limits bound each authenticated warehouse observation transaction. + +pub const MAX_FRESHNESS_RECORDS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs new file mode 100644 index 0000000000..8f170f2fce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -0,0 +1,103 @@ +//! Persist source identities; only legacy rows use a fixed observation-key fallback. + +use serde_json::json; +use sqlbuild_state::freshness::main::event_id::event_id as legacy_event_id; +use sqlbuild_state::freshness::models::FreshnessEventIdentity; +use uuid::Uuid; + +use crate::platform::main::require_instance::require_instance; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +pub async fn ingest_source_freshness( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &SourceFreshnessBatch, +) -> Result { + let invalid_identity = batch + .records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.source_name, + &record.run_id, + &record.strategy, + &record.value_kind, + &record.data_version_hash, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_source_id = batch.records.iter().any(|record| { + record + .event_id + .as_ref() + .is_some_and(|value| value.trim().is_empty()) + }); + if batch.generation < 1 + || batch.records.len() > MAX_FRESHNESS_RECORDS + || invalid_identity + || invalid_source_id + { + return Err(ControlError::Invalid(format!( + "source freshness sync needs a positive generation, at most {MAX_FRESHNESS_RECORDS} records and nonempty state identities", + ))); + } + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, batch.session_id, target), + ) + .await?; + let (mut transaction, rows) = query( + transaction, + "SELECT s.snapshot_id FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2 AND s.generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(batch.generation)], + ).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + let mut applied = false; + for record in &batch.records { + let source_event_id = record.event_id.clone().unwrap_or_else(|| { + legacy_event_id(&FreshnessEventIdentity { + source_name: &record.source_name, + target_database: record.target_database.as_deref(), + target_schema: record.target_schema.as_deref(), + target_name: record.target_name.as_deref(), + run_id: &record.run_id, + strategy: &record.strategy, + value_kind: &record.value_kind, + data_version_hash: &record.data_version_hash, + observed_at: &record + .observed_at + .format("%Y-%m-%dT%H:%M:%S%.6f+00:00") + .to_string(), + }) + }); + let identity = serde_json::to_vec(&( + target, + &record.state_database, + &record.state_schema, + "_sqlbuild_source_freshness", + &source_event_id, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&location_id, &identity); + let received_at = transaction.now; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at)]).await?; + applied |= !inserted.is_empty(); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs new file mode 100644 index 0000000000..0ff12b7aff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -0,0 +1 @@ +pub mod ingest_source_freshness; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs new file mode 100644 index 0000000000..69fb999ac1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs @@ -0,0 +1,5 @@ +//! Warehouse-owned observations mirrored by authenticated code locations, never execution state. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs new file mode 100644 index 0000000000..388464b8ab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -0,0 +1,56 @@ +//! Existing source-freshness state rows, with their physical state-table provenance. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Preserve observations, not an inferred latest version, age status or run outcome. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFreshnessRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub source_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub strategy: String, + pub value_kind: String, + pub data_version: Option, + pub data_version_hash: String, + pub observed_at: DateTime, +} + +/// A bounded page from one registered reader of a published location/target generation. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, +} + +/// Physical namespace of SQLBuild's source-freshness state table, never a source query. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseStateLocation { + pub database: Option, + pub schema: String, +} + +/// Private interpreter input; cursor values are data and receive adapter literal rendering. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 11]>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessPage { + pub records: Vec, + pub next_after: Option<[Option; 11]>, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 722e726650..2faba15f57 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -16,6 +16,8 @@ mod test_location_work; mod test_reads; #[path = "test_types.rs"] mod test_types; +#[path = "test_warehouse_metadata.rs"] +mod test_warehouse_metadata; #[path = "test_worker.rs"] mod test_worker; #[path = "work/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index bc54a92560..a89d419bcd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,8 +46,10 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 133e475a53..71c81b400d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -44,3 +44,9 @@ pub(crate) struct LocationWorkTestCase { pub(crate) backend: Backend, pub(crate) expected_generation: i64, } + +pub(crate) struct WarehouseMetadataTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs new file mode 100644 index 0000000000..efc31663b5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -0,0 +1,306 @@ +//! A scoped HTTP reader mirrors immutable warehouse observations, including external runs. + +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +#[test] +fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicity_are_preserved() { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite warehouse history", + backend: Backend::Sqlite, + expected_observations: 4, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL warehouse history", + backend: Backend::Postgres, + expected_observations: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let input = snapshot(location); + publish_snapshot(store, &input).await.expect("snapshot"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!( + "{}/api/locations/{location}/targets/development/metadata/source-freshness", + api.url + ); + let record = json!({ + "event_id": "warehouse-observation-1", + "state_database": null, "state_schema": "main", "source_name": "raw_orders", + "target_database": null, "target_schema": "main", "target_name": "orders", + "run_id": "external-command-1", "strategy": "max_column", "value_kind": "integer", + "data_version": "18446744073709551617", "data_version_hash": "same-data-version", + "observed_at": "2026-01-01T10:00:00Z", + }); + let mut later = record.clone(); + later["event_id"] = json!("warehouse-observation-2"); + later["observed_at"] = json!("2026-01-01T11:00:00Z"); + let mut other_run = record.clone(); + other_run + .as_object_mut() + .expect("record") + .remove("event_id"); + other_run["run_id"] = json!("external-command-2"); + let mut distinct = record.clone(); + distinct["event_id"] = json!("warehouse-observation-3"); + let mut page = vec![record.clone(); 100]; + page[1] = later; + page[2] = other_run.clone(); + page[3] = distinct; + // Both missing and explicitly NULL identities use the same fixed legacy fallback. + other_run["event_id"] = json!(null); + page[4] = other_run; + let batch = json!({"session_id": session_id, "generation": 1, "records": page}); + let outbound = connect_location_work((&api.url, &token.bearer), location) + .expect("pooled native location client"); + let typed: SourceFreshnessBatch = serde_json::from_value(batch.clone()).expect("page"); + for expected_applied in [true, false] { + assert_eq!( + outbound + .sync_source_freshness("development", &typed) + .await + .expect("native client sync"), + expected_applied + ); + } + let mut equivalent = batch.clone(); + equivalent["records"] = json!([record]); + equivalent["records"][0]["observed_at"] = json!("2026-01-01T11:00:00+01:00"); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&equivalent) + .send() + .await + .expect("equivalent timestamp") + .json::() + .await + .expect("reply"), + json!({"applied": false}) + ); + let mut stale = batch.clone(); + stale["generation"] = json!(2); + let mut invalid = batch.clone(); + invalid["records"][1]["source_name"] = json!(" "); + let mut invalid_id = batch.clone(); + invalid_id["records"][0]["event_id"] = json!(" "); + let mut oversized = batch.clone(); + oversized["records"] = json!(vec![record.clone(); 101]); + let mut unknown = batch.clone(); + unknown["records"][0]["status"] = json!("success"); + let foreign = url.replace(&location.to_string(), &other.to_string()); + let other_target = url.replace("/development/", "/production/"); + for (endpoint, body, expected) in [ + (&url, stale, StatusCode::CONFLICT), + (&url, invalid, StatusCode::BAD_REQUEST), + (&url, invalid_id, StatusCode::BAD_REQUEST), + (&url, oversized, StatusCode::BAD_REQUEST), + (&url, unknown, StatusCode::UNPROCESSABLE_ENTITY), + (&foreign, batch.clone(), StatusCode::FORBIDDEN), + (&other_target, batch.clone(), StatusCode::BAD_REQUEST), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(&token.bearer) + .json(&body) + .send() + .await + .expect("rejected page") + .status(), + expected + ); + } + assert_eq!( + client + .post(&url) + .body("invalid") + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "warehouse page commits all records or none", + table: "sqb_warehouse_freshness", + operation: "INSERT", + condition: "WHEN NEW.record_json->>'source_name'='inventory'", + sqlite_body: "SELECT RAISE(ABORT, 'injected metadata failure')", + postgres_body: "RAISE EXCEPTION 'injected metadata failure'", + }; + install_fault(store, &fault).await; + let mut new_record = record.clone(); + new_record["state_schema"] = json!("reporting"); + let mut inventory = new_record.clone(); + inventory["event_id"] = json!("inventory-observation-1"); + inventory["source_name"] = json!("inventory"); + let mut additional = batch.clone(); + additional["records"] = json!([new_record, inventory]); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("faulted page") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT record_json FROM sqb_warehouse_freshness ORDER BY observed_at,event_id", + vec![], + ) + .await + .expect("mirrored history"); + commit(transaction).await.expect("close read"); + assert_eq!( + rows.len(), + test_case.expected_observations, + "{}: no observation collapsed by data version and no partial page committed", + test_case.description + ); + for row in rows { + assert_eq!(row["record_json"]["data_version"], "18446744073709551617"); + assert_eq!(row["record_json"]["state_schema"], "main"); + } + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("retry after rollback") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + migrate_store(store) + .await + .expect("warm migration preserves observations"); + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT COUNT(*) AS n FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("retained history"); + commit(transaction).await.expect("close read"); + assert_eq!(rows[0]["n"], 6); + // The same warehouse rows in another target are separate observations, not replays. + let preview_session = Uuid::now_v7(); + let mut preview = input.clone(); + preview.target = "preview".into(); + publish_snapshot(store, &preview) + .await + .expect("preview snapshot"); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: preview_session, + target: "preview".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("preview reader"); + let mut preview_batch = batch.clone(); + preview_batch["session_id"] = json!(preview_session); + let preview_url = url.replace("/development/", "/preview/"); + assert_eq!( + client + .post(&preview_url) + .bearer_auth(&token.bearer) + .json(&preview_batch) + .send() + .await + .expect("separate target") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + "SELECT target, COUNT(*) AS n FROM sqb_warehouse_freshness GROUP BY target ORDER BY target", vec![]).await.expect("isolated history"); + commit(transaction).await.expect("close read"); + assert_eq!( + serde_json::to_value(rows).expect("rows"), + json!([{"target": "development", "n": 6}, {"target": "preview", "n": 4}]) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 545f2373f9..44a6f6f588 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,9 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_freshness", vec![]) + .await + .expect("remove subsequent warehouse mirror migration"); let (transaction, _) = query(transaction, "DROP INDEX sqb_node_finished_cursor", vec![]) .await diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 3464b72c6f..2badd5ab9b 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ + "DROP TABLE sqb_warehouse_freshness; \ + DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 7e4ed5ab92..6a71912e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -3,10 +3,10 @@ use fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION; use sqlbuild_analysis::contracts::constants::CONTRACT_ENFORCED; use sqlbuild_analysis::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, - NODE_TYPE_MODEL, NODE_TYPE_SEED, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; use sqlbuild_analysis::planner_changes::constants::{ AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, @@ -99,6 +99,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("COLUMN_DEFINITION_B64", COLUMN_DEFINITION_B64), ("COLUMN_METADATA_JSON_B64", COLUMN_METADATA_JSON_B64), ("COLUMN_TIMESTAMP", COLUMN_TIMESTAMP), + ("COLUMN_EVENT_ID", COLUMN_EVENT_ID), ("NODE_TYPE_MODEL", NODE_TYPE_MODEL), ("NODE_TYPE_SEED", NODE_TYPE_SEED), ("AUDIT_GATE_METADATA_KEY", AUDIT_GATE_METADATA_KEY), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 3a92672f90..b4612c1c7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; @@ -97,8 +98,13 @@ fn create_sql( )) } -/// The table DDL, adapter-rendered when the adapter overrides it, then any index DDL. -fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult> { +/// Create the table/indexes, then add the nullable identity column to a pre-existing table. +fn setup<'py>( + location: &Location<'_, 'py>, + writer: &Writer<'_, 'py>, + request: &Request<'_, 'py>, + describe: &dyn Fn(&PyErr) -> PyResult>, +) -> PyResult<()> { let py: Python<'_> = writer.execute.py(); let mut sqls: Vec = vec![match writer.render_create_table_sql { Some(render) => render @@ -118,7 +124,31 @@ fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult< sqls.push(sql?.extract()?); } } - Ok(sqls) + let database = location.database.into_pyobject(py)?.into_any(); + let schema = location.schema.into_pyobject(py)?.into_any(); + retry_write( + py, + || { + for sql in &sqls { + execute_sql(writer, sql)?; + } + ensure_event_id_column( + (writer.connection, writer.execute), + (&database, &schema), + (FINGERPRINT_TABLE_NAME, &qualified_table_name(location)?), + ( + &request + .optional("read_columns")? + .unwrap_or_else(|| py.None().into_bound(py)), + &request + .optional("render_add_columns")? + .unwrap_or_else(|| py.None().into_bound(py)), + writer.render_framework_type, + ), + ) + }, + describe, + ) } fn execute_sql<'py>(writer: &Writer<'_, 'py>, sql: &str) -> PyResult> { @@ -156,22 +186,14 @@ fn error_details<'py>(py: Python<'py>, error: &PyErr) -> PyResult( - writer: &Writer<'_, 'py>, - sqls: &[String], +/// Retry a complete warehouse operation after transient exceptions within the write budget. +fn retry_write<'py>( + py: Python<'py>, + write: impl Fn() -> PyResult<()>, describe: &dyn Fn(&PyErr) -> PyResult>, ) -> PyResult<()> { - let py: Python<'py> = writer.execute.py(); for attempt in 0..WRITE_ATTEMPTS { - let mut failure: Option = None; - for sql in sqls { - if let Err(error) = execute_sql(writer, sql) { - failure = Some(error); - break; - } - } - let Some(error) = failure else { + let Err(error) = write() else { return Ok(()); }; if attempt + 1 == WRITE_ATTEMPTS || !error.is_instance_of::(py) { @@ -376,16 +398,16 @@ fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult return Err(error), } } - let mut sqls: Vec = setup_sqls(&location, &writer)?; - sqls.push(insert); - execute_with_retry(&writer, &sqls, &|error| { + let describe = |error: &PyErr| { let mut details: Vec> = vec![ native_field(&fingerprint, "node_type")?, native_field(&fingerprint, "node_name")?, ]; details.extend(error_details(py, error)?); Ok((SINGLE_RETRY_MESSAGE, details)) - })?; + }; + setup(&location, &writer, &request, &describe)?; + retry_write(py, || execute_sql(&writer, &insert).map(|_| ()), &describe)?; Ok(true) } @@ -420,8 +442,7 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let on_progress: Option> = present(&request, "on_progress")?; let parts: WriteParts<'py> = WriteParts::read(&request)?; let (location, writer) = (parts.location(), parts.writer()); - let setup: Vec = setup_sqls(&location, &writer)?; - execute_with_retry(&writer, &setup, &|error| { + setup(&location, &writer, &request, &|error| { batch_warning(&fingerprints, error) })?; let batches = fingerprint_insert_batches( @@ -434,9 +455,11 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let mut completed: usize = 0; for (batch, insert_sql) in batches { let members: Vec> = batch.iter().collect(); - execute_with_retry(&writer, &[insert_sql], &|error| { - batch_warning(&members, error) - })?; + retry_write( + py, + || execute_sql(&writer, &insert_sql).map(|_| ()), + &|error| batch_warning(&members, error), + )?; completed += members.len(); if let Some(progress) = &on_progress { let keywords: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index ab91f568ae..598be15e62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -177,6 +177,11 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { "render_create_index_sqls", native_field(adapter, "render_create_source_freshness_index_sqls")?, ), + ("read_columns", native_field(adapter, "get_columns")?), + ( + "render_add_columns", + native_field(adapter, "render_add_columns")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 292bada80f..4a004cbbda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -4,7 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; -use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::constants::{ + REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, + STRING_STATE_VALUE_TYPE, +}; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; @@ -13,6 +16,7 @@ use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, }; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -82,12 +86,20 @@ fn create_table_sql<'py>( ) -> PyResult> { let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; - let columns: Bound<'py, PyAny> = - PyTuple::new(py, SOURCE_FRESHNESS_COLUMN_TYPES.map(|(column, _)| column))?.into_any(); + let columns: Bound<'py, PyAny> = PyTuple::new( + py, + SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .chain([SOURCE_FRESHNESS_EVENT_ID]) + .collect::>(), + )? + .into_any(); let column_types: Bound<'py, PyDict> = PyDict::new(py); for (column, declared) in SOURCE_FRESHNESS_COLUMN_TYPES { column_types.set_item(column, declared)?; } + column_types.set_item(SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE)?; let column_types: Bound<'py, PyAny> = column_types.into_any(); let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); let sql: String = state_table_create_sql( @@ -180,7 +192,7 @@ freshness table to regenerate source freshness state." ) } -fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = row.py(); let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { @@ -306,6 +318,20 @@ pub(crate) fn write_source_freshness_records<'py>( call_with(&render_create, &location())? }; execute_sql((execute, connection), create_sql)?; + let qualified = qualified_table_name( + &native_field(renderers, "render_qualified_name")?, + (database, schema), + )?; + ensure_event_id_column( + (connection, execute), + (database, schema), + (SOURCE_FRESHNESS_TABLE_NAME, &qualified), + ( + &native_field(renderers, "read_columns")?, + &native_field(renderers, "render_add_columns")?, + &native_field(renderers, "render_framework_type")?, + ), + )?; let render_indexes: Bound<'py, PyAny> = native_field(renderers, "render_create_index_sqls")?; if !render_indexes.is_none() { for index_sql in call_with(&render_indexes, &location())?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 7a5f85c971..1ca8f40504 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -6,11 +6,13 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::warehouse_metadata::models::SourceFreshnessRead; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_source_freshness; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; @@ -44,6 +46,24 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul } }; serde_json::to_string(&record) + } else if request.get("source_freshness").is_some() { + let metadata: SourceFreshnessRead = + serde_json::from_value(request["source_freshness"].clone()).map_err(|_| { + PyValueError::new_err("runtime received an invalid metadata read") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + let page = match read_source_freshness(py, &arguments, &metadata) { + Ok(page) => Some(page), + Err(_) => { + write_console( + py, + "stderr", + "Warehouse metadata read failed. No page delivered.\n", + )?; + None + } + }; + serde_json::to_string(&page) } else { let evaluation: SensorEvaluationRequest = serde_json::from_value(request).map_err(|_| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index cb882f735f..248bf8cc95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -13,3 +13,5 @@ pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; pub(crate) mod sqlite_cursor; +pub(crate) mod state_column_evolution; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs new file mode 100644 index 0000000000..bfb47dcba2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs @@ -0,0 +1,85 @@ +//! Warehouse catalog and nullable-column DDL adapter edge for persisted event identity. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field; + +/// Missing callbacks retain the existing renderer-only API for already-current/new schemas. +pub(crate) fn ensure_event_id_column<'py>( + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (table, qualified): (&str, &str), + (read_columns, render_add, render_type): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<()> { + if read_columns.is_none() && render_add.is_none() { + return Ok(()); + } + if read_columns.is_none() || render_add.is_none() { + return Err(PyValueError::new_err( + "state column evolution requires both read_columns and render_add_columns", + )); + } + let py = connection.py(); + let scope = [ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", PyString::new(py, table).into_any()), + ]; + if event_id_present(read_columns, &scope)? { + return Ok(()); + } + let string_type = render_type.call1((field( + &host::entry(py, AdapterHost::FrameworkType)?, + "STRING", + )?,))?; + let column = call_with( + &host::entry(py, AdapterHost::ColumnInfo)?, + &[ + ("name", PyString::new(py, "event_id").into_any()), + ("type", string_type), + ], + )?; + let statements = call_with( + render_add, + &[ + ("destination", PyString::new(py, qualified).into_any()), + ("columns", PyTuple::new(py, [column])?.into_any()), + ], + )?; + for statement in statements.try_iter()? { + if let Err(error) = call_with( + execute, + &[("connection", connection.clone()), ("sql", statement?)], + ) && !event_id_present(read_columns, &scope).unwrap_or(false) + { + return Err(error); + } + } + Ok(()) +} + +/// Concurrent additions are harmless only once the authoritative catalog confirms the column. +fn event_id_present( + read_columns: &Bound<'_, PyAny>, + scope: &[(&str, Bound<'_, PyAny>)], +) -> PyResult { + for column in call_with(read_columns, scope)?.try_iter()? { + if field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id") + { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs new file mode 100644 index 0000000000..7be0ce4f6d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -0,0 +1,205 @@ +//! Read-only warehouse adapter edge; never compile declarations or write warehouse state. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use serde_json::{Value, json}; +use sqlbuild_config::command_connection::constants::{ + DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, +}; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::models::SqlValue; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + SourceFreshnessPage, SourceFreshnessRead, WarehouseFreshnessRecord, +}; +use sqlbuild_state::freshness::constants::{OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::main::read_history_sql::read_history_sql; + +use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::freshness::state_io::{qualified_table_name, row_record}; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::records::fields::field; + +pub(crate) fn read_source_freshness( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + request: &SourceFreshnessRead, +) -> PyResult { + if request.location.schema.trim().is_empty() { + return Err(PyValueError::new_err( + "metadata state schema must not be blank", + )); + } + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; + let connection_config = project_connection_config( + &discovered, + (&project_dir, &arguments.value("selected_target")?), + (&py.None().into_bound(py), true), + )?; + let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; + if let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter_name) + { + let config = connection_config.cast::()?; + let path: Option = config + .get_item(path_key)? + .map(|path| path.extract()) + .transpose()?; + if path + .as_deref() + .is_some_and(|path| path != DUCKDB_MEMORY_DATABASE) + { + config.set_item("read_only", true)?; + } + if adapter_name == DUCKDB_ADAPTER + && let Some(attachments) = config.get_item("attach")? + { + for attachment in attachments.try_iter()? { + attachment?.set_item("read_only", true)?; + } + } + } + let connection = adapter.call_method1("connect", (connection_config,))?; + let read = read_page(py, (&adapter, &connection), request); + let closed = adapter.call_method1("close", (&connection,)); + let page = read?; + closed?; + Ok(page) +} + +fn read_page( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &SourceFreshnessRead, +) -> PyResult { + let database = request.location.database.as_ref().map_or_else( + || py.None().into_bound(py), + |value| PyString::new(py, value).into_any(), + ); + let schema = PyString::new(py, &request.location.schema).into_any(); + let name = PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(); + let exists = call_with( + &adapter.getattr("relation_exists")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name), + ], + )? + .is_truthy()?; + if !exists { + return Ok(SourceFreshnessPage { + records: Vec::new(), + next_after: None, + }); + } + let columns = call_with( + &adapter.getattr("get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ( + "name", + PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(), + ), + ], + )?; + let mut has_event_id = false; + for column in columns.try_iter()? { + has_event_id |= field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id"); + } + let mut literals: [Option; 11] = Default::default(); + if let Some(after) = &request.after { + let classes = SqlValueClasses::load(py)?; + for (output, value) in literals.iter_mut().zip(after) { + if let Some(value) = value { + let value = sql_value(&SqlValue::String(value.clone()), &classes)?; + *output = Some( + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract()?, + ); + } + } + } + let qualified = qualified_table_name( + &adapter.getattr("render_qualified_name")?, + (&database, &schema), + )?; + let adapter_name = adapter.getattr("adapter_name")?.extract::()?; + let sql = read_history_sql( + &qualified, + request.after.as_ref().map(|_| &literals), + (MAX_FRESHNESS_RECORDS, &adapter_name, has_event_id), + ) + .map_err(PyValueError::new_err)?; + let result = call_with( + &adapter.getattr("query")?, + &[ + ("connection", connection.clone()), + ("sql", PyString::new(py, &sql).into_any()), + ( + "limit", + pyo3::IntoPyObjectExt::into_bound_py_any(MAX_FRESHNESS_RECORDS, py)?, + ), + ], + )?; + let mut records: Vec = Vec::new(); + let mut next_after = None; + for row in field(&result, "rows")?.try_iter()? { + let row = row?; + let record = row_record(&row)?; + let mut cursor: [Option; 11] = Default::default(); + let mut value = json!({"state_database": request.location.database, "state_schema": request.location.schema}); + for (index, (name, _)) in SOURCE_FRESHNESS_COLUMN_TYPES.iter().enumerate() { + let raw = row.get_item(index)?; + cursor[index] = if raw.is_none() { + None + } else { + Some(raw.str()?.extract()?) + }; + let item = field(&record, name)?; + value[*name] = if item.is_none() { + Value::Null + } else if *name == OBSERVED_AT_COLUMN { + json!(item.call_method0("isoformat")?.extract::()?) + } else { + json!(item.extract::()?) + }; + } + cursor[10] = row.get_item(10)?.extract::>()?; + value["event_id"] = json!(cursor[10]); + records.push( + serde_json::from_value::(value) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + next_after = Some(cursor); + } + if records.len() < MAX_FRESHNESS_RECORDS { + next_after = None; + } + Ok(SourceFreshnessPage { + records, + next_after, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 6a1c04bac6..3fdd402cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1367,6 +1367,8 @@ const SOURCE_FRESHNESS_RENDERERS: RecordSpec = RecordSpec { FieldSpec::required("render_insert_records_sql"), FieldSpec::new("render_create_table_sql", FieldDefault::None), FieldSpec::new("render_create_index_sqls", FieldDefault::None), + FieldSpec::new("read_columns", FieldDefault::None), + FieldSpec::new("render_add_columns", FieldDefault::None), ], }; const SOURCE_FRESHNESS_OBSERVATION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index 2a93fd2855..b02904a131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -14,7 +14,9 @@ use sqlbuild_state::freshness::main::age_status::age_status; use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; -use sqlbuild_state::freshness::models::{AgeStatus, ComparableRecord, DataVersion}; +use sqlbuild_state::freshness::models::{ + AgeStatus, ComparableRecord, DataVersion, FreshnessEventIdentity, +}; use sqlbuild_state::freshness::types::TimestampParser; use crate::bindings::_helpers::boundary::python_calls::imported; @@ -31,6 +33,40 @@ const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; const STRING_KIND: &str = "string"; +/// Normalize timestamp offsets and precision before computing a persisted observation identity. +pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { + let observed_at = presumed_utc(&native_field(record, "observed_at")?)?; + let observed_at = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}+00:00", + observed_at.date.year, + observed_at.date.month, + observed_at.date.day, + observed_at.hour, + observed_at.minute, + observed_at.second, + observed_at.microsecond, + ); + Ok(sqlbuild_state::freshness::main::event_id::event_id( + &FreshnessEventIdentity { + source_name: &native_field(record, "source_name")?.extract::()?, + target_database: native_field(record, "target_database")? + .extract::>()? + .as_deref(), + target_schema: native_field(record, "target_schema")? + .extract::>()? + .as_deref(), + target_name: native_field(record, "target_name")? + .extract::>()? + .as_deref(), + run_id: &native_field(record, "run_id")?.extract::()?, + strategy: &native_field(record, "strategy")?.extract::()?, + value_kind: &native_field(record, "value_kind")?.extract::()?, + data_version_hash: &native_field(record, "data_version_hash")?.extract::()?, + observed_at: &observed_at, + }, + )) +} + fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { match registry::native(py, class).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index 40a1d39bf6..9f7aca6b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -9,7 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::constants::{ - SOURCE_FRESHNESS_COLUMN_TYPES, STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE, + SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE, + TEXT_TIMESTAMP_STATE_VALUE_TYPE, }; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; @@ -20,6 +21,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::state_rows::freshness::record_event_id; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; /// Each state value type a state table may declare and the framework type that renders it. @@ -121,18 +123,20 @@ fn render_insert_source_freshness_records_sql<'py>( let mut rows: Vec> = Vec::new(); for record in records.try_iter()? { let record: Bound<'py, PyAny> = record?; - let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES + let mut row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, declared)| { render_state_sql_literal(py, &native_field(&record, column)?, declared, None) }) .collect::>()?; + row.push(format!("'{}'", record_event_id(&record)?)); rows.push(row); } - let names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + let mut names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, _)| *column) .collect(); + names.push(SOURCE_FRESHNESS_EVENT_ID); Ok(values_insert_sql( &table.extract::()?, &names, diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 8083997c83..4bec5268c8 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -41,7 +41,7 @@ pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ pub const STRING_STATE_VALUE_TYPE: &str = "string"; /// State value type of a timestamp stored as ISO text. pub const TEXT_TIMESTAMP_STATE_VALUE_TYPE: &str = "text_timestamp"; -/// Source freshness state table columns in table order, with the state value type each declares. +/// Legacy latest-reader columns; the nullable event ID is intentionally not required by reads. pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("source_name", STRING_STATE_VALUE_TYPE), ("target_database", STRING_STATE_VALUE_TYPE), @@ -54,5 +54,8 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("data_version_hash", STRING_STATE_VALUE_TYPE), ("observed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), ]; +pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/event_id.rs b/crates/sqlbuild-state/src/freshness/main/event_id.rs new file mode 100644 index 0000000000..72bdeab470 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/event_id.rs @@ -0,0 +1,22 @@ +//! Observation identity excludes mutable payload and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::freshness::models::FreshnessEventIdentity; + +/// `observed_at` is UTC ISO text with six fractional digits and a `+00:00` offset. +pub fn event_id(row: &FreshnessEventIdentity<'_>) -> String { + let identity = serde_json::json!([ + "sqlbuild.source-freshness.event.v1", + row.source_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.observed_at, + row.strategy, + row.value_kind, + row.data_version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 9ec47ed663..5a1085a0e2 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -1,7 +1,9 @@ pub mod age_status; pub mod configuration_error; pub mod data_version_hash; +pub mod event_id; pub mod normalized_data_version; +pub mod read_history_sql; pub mod read_latest_sql; pub mod records_equivalent; pub mod relevant_sources_sql; diff --git a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs new file mode 100644 index 0000000000..ce7df1f246 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs @@ -0,0 +1,112 @@ +//! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. + +use crate::freshness::constants::{ + OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, SQLSERVER_ADAPTER, +}; + +/// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. +pub fn read_history_sql( + qualified_name: &str, + after: Option<&[Option; 11]>, + (limit, adapter, has_event_id): (usize, &str, bool), +) -> Result { + let mut columns: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(name, _)| *name) + .collect(); + let mut projection = columns.join(", "); + projection.push_str(if has_event_id { + ", event_id" + } else { + ", NULL AS event_id" + }); + columns.push(if has_event_id { + SOURCE_FRESHNESS_EVENT_ID + } else { + "NULL" + }); + let ordered = columns + .iter() + .map(|column| { + if *column == OBSERVED_AT_COLUMN { + Ok((*column).to_owned()) + } else { + binary(column, adapter) + } + }) + .collect::, _>>()?; + let mut greater: Vec = Vec::new(); + let mut equal: Vec = Vec::new(); + if let Some(after) = after { + for ((column, ordered), value) in columns.iter().zip(&ordered).zip(after) { + let literal = value + .as_ref() + .map(|value| { + if *column == OBSERVED_AT_COLUMN { + Ok(value.clone()) + } else { + binary(value, adapter) + } + }) + .transpose()?; + let comparison = literal.as_ref().map_or_else( + || format!("{column} IS NOT NULL"), + |literal| format!("{ordered} > {literal}"), + ); + let prefix = if equal.is_empty() { + String::new() + } else { + format!("{} AND ", equal.join(" AND ")) + }; + greater.push(format!("({prefix}{comparison})")); + equal.push(literal.as_ref().map_or_else( + || format!("{column} IS NULL"), + |literal| format!("{ordered} = {literal}"), + )); + } + } + let filter = if greater.is_empty() { + String::new() + } else { + format!(" WHERE {}", greater.join(" OR ")) + }; + let order: Vec = columns + .iter() + .zip(&ordered) + .map(|(column, ordered)| format!("CASE WHEN {column} IS NULL THEN 0 ELSE 1 END, {ordered}")) + .collect(); + let top = adapter == SQLSERVER_ADAPTER; + let head = if top { + format!("TOP {limit} ") + } else { + String::new() + }; + let tail = if top { + String::new() + } else { + format!(" LIMIT {limit}") + }; + Ok(format!( + "SELECT {head}{projection} FROM {qualified_name}{filter} ORDER BY {}{tail}", + order.join(", ") + )) +} + +/// Session and column collations must not merge case, accents or trailing spaces at page boundaries. +fn binary(value: &str, adapter: &str) -> Result { + Ok(match adapter { + "duckdb" | "motherduck" => format!("encode({value})"), + "sqlite" => format!("CAST({value} AS BLOB)"), + "postgres" => format!("convert_to({value}, 'UTF8')"), + "snowflake" => format!("TO_BINARY({value}, 'UTF-8')"), + "bigquery" => format!("CAST({value} AS BYTES)"), + "databricks" => format!("CAST({value} AS BINARY)"), + "sqlserver" => format!("CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {value}))"), + "clickhouse" => value.to_owned(), + _ => { + return Err(format!( + "source freshness history does not support adapter '{adapter}'" + )); + } + }) +} diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs index dbe1341a71..9f7cbfb958 100644 --- a/crates/sqlbuild-state/src/freshness/models.rs +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -2,6 +2,21 @@ use sqlbuild_cursor::algebra::models::LocalDateTime; +/// Stable identity fields shared by warehouse writers and legacy history ingestion. +#[derive(Debug)] +pub struct FreshnessEventIdentity<'a> { + pub source_name: &'a str, + pub target_database: Option<&'a str>, + pub target_schema: Option<&'a str>, + pub target_name: Option<&'a str>, + pub run_id: &'a str, + pub strategy: &'a str, + pub value_kind: &'a str, + pub data_version_hash: &'a str, + /// UTC ISO text with six fractional digits and a `+00:00` offset. + pub observed_at: &'a str, +} + /// An observed data version of one value kind. #[derive(Clone, Debug, PartialEq, Eq)] pub enum DataVersion { diff --git a/fensu.toml b/fensu.toml index 16d15444db..5a13ca223d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2657,6 +2657,7 @@ module_layers = [ "native_host/model_lookup=project", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", + "native_host/warehouse_metadata=commands", "native_threads=infra", "output_capture=output", "plan_command=commands", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 84509d6ae7..31832558f1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -199,6 +199,7 @@ COLUMN_SCHEMA_FINGERPRINT: str COLUMN_DEFINITION_B64: str COLUMN_METADATA_JSON_B64: str COLUMN_TIMESTAMP: str +COLUMN_EVENT_ID: str NODE_TYPE_MODEL: str NODE_TYPE_SEED: str AUDIT_GATE_METADATA_KEY: str @@ -4187,6 +4188,8 @@ def write_fingerprint( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, ensure_table: bool = True, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> bool: ... def write_fingerprints( *, @@ -4200,6 +4203,8 @@ def write_fingerprints( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, on_progress: FingerprintWriteProgress | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... def select_latest_fingerprints( *, @@ -7838,6 +7843,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str] render_create_table_sql: Callable[..., str] | None render_create_index_sqls: Callable[..., tuple[str, ...]] | None + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None + render_add_columns: Callable[..., tuple[str, ...]] | None def __init__( self, render_qualified_name: Callable[..., str | None], @@ -7845,6 +7852,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str], render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... class SourceFreshnessObservation(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index d376a6a228..f6d52b95c4 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1080,7 +1080,9 @@ def connect(self, config: dict[str, Any]) -> Any: import duckdb database: str = str(config.get("database", ":memory:")) - raw_connection: duckdb.DuckDBPyConnection = duckdb.connect(database=database) + raw_connection: duckdb.DuckDBPyConnection = duckdb.connect( + database=database, read_only=bool(config.get("read_only", False)) + ) return self._configured_connection(raw_connection=raw_connection, config=config) def connect_local_replay(self, database: Path) -> Any: diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2f200b531d..25d13ae0f1 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -340,6 +340,7 @@ def render_create_fingerprint_table_sql( from sqlbuild._native import ( COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, + COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, COLUMN_NODE_TYPE, @@ -373,7 +374,8 @@ def render_create_fingerprint_table_sql( f"{COLUMN_SCHEMA_FINGERPRINT} NVARCHAR(450) NOT NULL, " f"{COLUMN_DEFINITION_B64} NVARCHAR(MAX) NOT NULL, " f"{COLUMN_METADATA_JSON_B64} NVARCHAR(MAX) NOT NULL, " - f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL" + f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL, " + f"{COLUMN_EVENT_ID} NVARCHAR(MAX)" f")" ) return self._create_table_if_missing_sql( diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 5c2abd241e..0378d85973 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -205,6 +205,8 @@ def _on_group_progress( fingerprints=group, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, on_progress=_on_group_progress, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index d8d0c9f088..f4559e8027 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -118,6 +118,8 @@ def materialize_query_diff_artifact( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index a95ec0cd3b..c7f935e162 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -182,6 +182,8 @@ def _try_write_function_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index edc76da34c..310f994670 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -55,6 +55,8 @@ def try_write_python_node_identity_fingerprint( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/review/_helpers/marker.py b/src/sqlbuild/executor/review/_helpers/marker.py index ee87afbfbf..1f2f23e173 100644 --- a/src/sqlbuild/executor/review/_helpers/marker.py +++ b/src/sqlbuild/executor/review/_helpers/marker.py @@ -52,6 +52,8 @@ def write_review_marker( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index e084b6ab5d..065fe30123 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -97,6 +97,8 @@ def try_write_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index a103b8bb6b..02a44cc063 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -56,6 +56,8 @@ def try_write_seed_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index ee942f38ab..681b669497 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,5 +1,6 @@ """Publication cases and isolated server credentials.""" +from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path from subprocess import Popen @@ -143,3 +144,13 @@ class CalendarRejectionTestCase: class CalendarRefreshTestCase: description: str expected_timezone: str + + +@dataclass(frozen=True) +class WarehouseHistoryTestCase: + description: str + adapter_name: str + expected_page_lengths: tuple[int, ...] + connection_key: str + journal_modes: tuple[str, ...] + append_fingerprints: Callable[..., None] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 8f5aca9f03..16170f6d72 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -12,7 +12,7 @@ import sys import time from collections.abc import Callable, Iterator -from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress +from contextlib import AbstractContextManager, closing, contextmanager, nullcontext, suppress from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path @@ -25,6 +25,7 @@ import duckdb import pytest +from sqlbuild._native import Fingerprint, write_fingerprint from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -590,3 +591,56 @@ def sensor_calls(path: Path, fields: dict[str, Any]) -> list[dict[str, Any]]: """Select recorded callback invocations without inferring execution from declarations.""" records: list[dict[str, Any]] = [json.loads(line) for line in path.read_text().splitlines()] return list(filter(lambda record: fields.items() <= record.items(), records)) + + +def append_retried_fingerprints( + *, + adapter: Any, + connection: Any, + database_path: Path, + fingerprint: Fingerprint, + read_columns: Callable[..., Any] | None = None, +) -> None: + """Retry the same observation without substituting its identity or live catalog.""" + del database_path + for _attempt in range(2): + _ = write_fingerprint( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprint=fingerprint, + ensure_table=False, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + read_columns=read_columns or adapter.get_columns, + render_add_columns=adapter.render_add_columns, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + ) + + +def append_contended_fingerprints( + *, adapter: Any, connection: Any, database_path: Path, fingerprint: Fingerprint +) -> None: + """Release a real SQLite write lock only after the first ALTER has failed.""" + with closing(sqlite3.connect(database_path)) as blocker: + _ = adapter.execute(connection=connection, sql="PRAGMA busy_timeout=1") + catalog_actions: Iterator[Callable[[], Any]] = iter( + (lambda: blocker.execute("BEGIN IMMEDIATE"), blocker.commit) + ) + + def read_columns(**scope: Any) -> Any: + columns: Any = adapter.get_columns(**scope) + next(catalog_actions, lambda: None)() + return columns + + try: + append_retried_fingerprints( + adapter=adapter, + connection=connection, + database_path=database_path, + fingerprint=fingerprint, + read_columns=read_columns, + ) + finally: + blocker.rollback() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py new file mode 100644 index 0000000000..df565a66a6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -0,0 +1,293 @@ +"""Real warehouse writers, populated predecessors and the isolated read-only history protocol.""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from contextlib import closing +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + Fingerprint, + SourceFreshnessRecord, + SourceFreshnessRenderers, + read_latest_fingerprints, + read_latest_source_freshness, + write_source_freshness_records, +) +from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WarehouseHistoryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + append_contended_fingerprints, + append_retried_fingerprints, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseHistoryTestCase( + "DuckDB typed timestamps", + "duckdb", + (100, 6), + "database", + (), + append_retried_fingerprints, + ), + WarehouseHistoryTestCase( + "SQLite timestamp text", + "sqlite", + (100, 6), + "path", + ("delete",), + append_contended_fingerprints, + ), + ], + ids=lambda case: case.description, +) +def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_preserves_new_legacy_and_retried_observations( + test_case: WarehouseHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + connection_key: str = test_case.connection_key + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + f'name = "orders"\nadapter = "{test_case.adapter_name}"\n' + f'[connections.local]\n{connection_key} = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("metadata must not compile user declarations")\n', + }, + ) + adapter: Any = resolve_adapter(adapter_name=test_case.adapter_name) + connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) + legacy_sql: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" + ).read_text() + for statement in legacy_sql.rstrip().removesuffix(";").split(";"): + _ = adapter.execute(connection=connection, sql=statement) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + + def page(after: list[str | None] | None = None) -> Any: + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "runtime", + "--server", + "http://127.0.0.1:1", + "--location-id", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + ], + input=json.dumps( + { + "source_freshness": { + "location": {"database": None, "schema": "main"}, + "after": after, + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + old: dict[str, Any] = page() + assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + connection = adapter.connect({connection_key: str(project / "warehouse.db")}) + # Reading history never upgrades the warehouse or imports project declarations. + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_source_freshness" + ) + } + latest: Any = read_latest_fingerprints( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, + ) + assert latest.fingerprints["orders"].run_id == "legacy-run" + fingerprint: Fingerprint = Fingerprint( + node_type="model", + node_name="orders", + target_database=None, + target_schema="main", + target_name="orders", + run_id="external-run", + definition_hash="definition", + version_hash="version", + schema_fingerprint="schema", + definition="SELECT 2 AS order_id", + ts=datetime(2026, 2, 1, tzinfo=UTC), + ) + test_case.append_fingerprints( + adapter=adapter, + connection=connection, + database_path=project / "warehouse.db", + fingerprint=fingerprint, + ) + expected_fingerprint_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.fingerprint.event.v1", + "model", + "orders", + None, + "main", + "orders", + "external-run", + "version", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_fingerprints ORDER BY ts", + limit=10, + ).rows == ( + (None,), + (expected_fingerprint_id,), + (expected_fingerprint_id,), + ) + records: tuple[SourceFreshnessRecord, ...] = tuple( + SourceFreshnessRecord( + source_name=("Orders", "orders", "orders ", "ordérs")[index % 4], + target_database=("", None)[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + strategy="max_column", + value_kind="string", + data_version=("o'rders 🧾", None, None)[index % 3], + data_version_hash=f"value-{index}", + observed_at=datetime(2026, 2, 1, microsecond=index, tzinfo=UTC), + ) + for index in range(104) + ) + renderers: SourceFreshnessRenderers = SourceFreshnessRenderers( + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + for batch in (records, records[:1]): + write_source_freshness_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=batch, + renderers=renderers, + ) + expected_freshness_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.source-freshness.event.v1", + "Orders", + "", + "main", + "orders", + "external-0", + "2026-02-01T00:00:00.000000+00:00", + "max_column", + "string", + "value-0", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_source_freshness WHERE run_id='external-0'", + limit=10, + ).rows == ( + (expected_freshness_id,), + (expected_freshness_id,), + ) + # An equivalent offset denotes the same observation, not another event. + equivalent: SourceFreshnessRecord = SourceFreshnessRecord( + source_name="Orders", + target_database="", + target_schema="main", + target_name="orders", + run_id="external-0", + strategy="max_column", + value_kind="string", + data_version="o'rders 🧾", + data_version_hash="value-0", + observed_at=datetime(2026, 2, 1, 1, tzinfo=timezone(timedelta(hours=1))), + ) + assert expected_freshness_id in adapter.render_insert_source_freshness_records_sql( + database=None, schema="main", records=(equivalent,) + ) + current: Any = read_latest_source_freshness( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, + ) + assert {row.run_id for row in current.records.values()} == { + "legacy-run", + "external-100", + "external-101", + "external-102", + "external-103", + } + adapter.close(connection) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == test_case.expected_page_lengths + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + event_ids: set[str | None] = {row["event_id"] for row in imported} + assert len(event_ids - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == 1 + assert {row["run_id"] for row in imported} == {"legacy-run"} | { + f"external-{index}" for index in range(104) + } + assert {row["target_database"] for row in imported} == {None, ""} + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() From 7031203d50ea418f84e17739a7f814a15d47e292 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:23:24 +0000 Subject: [PATCH 056/138] test: name fingerprint columns in stale-row fixture Preserve the legacy stale-row planning regression after additive warehouse schema evolution. Leave the nullable event identity absent and keep every query-change assertion unchanged. --- .../cli/commands/main/build/test_query_change_tracking.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py index 4fe9306891..ebb83d9c96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py @@ -146,7 +146,10 @@ def test_given_unchanged_project_when_planning_after_build_then_models_are_not_q try: stale_row: tuple[object, ...] = tuple(fingerprint_rows[0]) connection.execute( - "INSERT INTO main._sqlbuild_fingerprints VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO main._sqlbuild_fingerprints " + "(node_type, node_name, target_database, target_schema, target_name, run_id, " + "definition_hash, version_hash, schema_fingerprint, definition_b64, " + "metadata_json_b64, ts) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( stale_row[0], stale_row[1], From d12cf2741c2e360c148c9195f9b237a59febcc5d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:23:25 +0000 Subject: [PATCH 057/138] feat: bound freshness history by observation time Keep an inclusive observation-time bound separate from row-key pagination. Admit legacy SQLite timestamp formats through a date-only SQL superset, enforce the exact decoded UTC bound, and retain continuation after an empty filtered page. Exercise populated predecessor schemas, offset and microsecond boundaries, retries and both local warehouse backends without changing latest readers or adding producer policy. --- crates/README.md | 11 +++-- .../src/warehouse_metadata/models.rs | 2 + .../native_host/warehouse_metadata.rs | 40 ++++++++++++++++--- .../sqlbuild-state/src/freshness/constants.rs | 1 + .../src/freshness/main/read_history_sql.rs | 12 +++++- .../publication/test_warehouse_metadata.py | 38 +++++++++++++++++- 6 files changed, 92 insertions(+), 12 deletions(-) diff --git a/crates/README.md b/crates/README.md index 41952bc041..03c551e8d2 100644 --- a/crates/README.md +++ b/crates/README.md @@ -128,9 +128,14 @@ The location history reader is read-only, including local database connections: databases nor evolves tables. Source-freshness ingestion preserves warehouse IDs, scoped by location, target and physical state table. Only missing or NULL IDs use the fixed legacy observation tuple, never a hash of the evolving full row. Bounded pages commit atomically under the current location -session and snapshot generation, and repeated pages are idempotent. These are read/ingestion -primitives; automatic polling, watermark/overlap management and deletion reconciliation are separate -location-sync responsibilities, not implemented by the page reader. +session and snapshot generation, and repeated pages are idempotent. A history read can hold an +inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. +SQLite's SQL prefilter includes the boundary day so legacy space-separated timestamps and variable +fractional precision remain readable; decoded UTC timestamps enforce the exact bound. Other adapters +prefilter at whole-second precision. Continuation follows the raw row count: even an empty decoded +page can have more rows to inspect. These are read/ingestion primitives; automatic polling, +watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, +not implemented by the page reader. ## Working on the crates diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 388464b8ab..13ac7c9600 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -46,6 +46,8 @@ pub struct WarehouseStateLocation { pub struct SourceFreshnessRead { pub location: WarehouseStateLocation, pub after: Option<[Option; 11]>, + /// Inclusive time bound, held fixed across pages; independent of the row-key cursor. + pub observed_since: Option>, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 7be0ce4f6d..3422a7d6f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -13,7 +13,9 @@ use sqlbuild_orchestration::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS use sqlbuild_orchestration::warehouse_metadata::models::{ SourceFreshnessPage, SourceFreshnessRead, WarehouseFreshnessRecord, }; -use sqlbuild_state::freshness::constants::{OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::constants::{ + OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, +}; use sqlbuild_state::freshness::main::read_history_sql::read_history_sql; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; @@ -147,9 +149,29 @@ fn read_page( (&database, &schema), )?; let adapter_name = adapter.getattr("adapter_name")?.extract::()?; + let observed_since = request + .observed_since + .map(|since| { + let text = since + .format(if adapter_name == SQLITE_ADAPTER { + "%Y-%m-%d" + } else { + "%Y-%m-%dT%H:%M:%S+00:00" + }) + .to_string(); + let classes = SqlValueClasses::load(py)?; + let value = sql_value(&SqlValue::String(text), &classes)?; + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract::() + }) + .transpose()?; let sql = read_history_sql( &qualified, request.after.as_ref().map(|_| &literals), + observed_since.as_deref(), (MAX_FRESHNESS_RECORDS, &adapter_name, has_event_id), ) .map_err(PyValueError::new_err)?; @@ -166,7 +188,9 @@ fn read_page( )?; let mut records: Vec = Vec::new(); let mut next_after = None; + let mut read_count = 0; for row in field(&result, "rows")?.try_iter()? { + read_count += 1; let row = row?; let record = row_record(&row)?; let mut cursor: [Option; 11] = Default::default(); @@ -189,13 +213,17 @@ fn read_page( } cursor[10] = row.get_item(10)?.extract::>()?; value["event_id"] = json!(cursor[10]); - records.push( - serde_json::from_value::(value) - .map_err(|error| PyValueError::new_err(error.to_string()))?, - ); + let record = serde_json::from_value::(value) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if request + .observed_since + .is_none_or(|since| record.observed_at >= since) + { + records.push(record); + } next_after = Some(cursor); } - if records.len() < MAX_FRESHNESS_RECORDS { + if read_count < MAX_FRESHNESS_RECORDS { next_after = None; } Ok(SourceFreshnessPage { diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 4bec5268c8..0dc528ac8c 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -56,6 +56,7 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ]; pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +pub const SQLITE_ADAPTER: &str = "sqlite"; pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs index ce7df1f246..e894646829 100644 --- a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs @@ -8,6 +8,7 @@ use crate::freshness::constants::{ pub fn read_history_sql( qualified_name: &str, after: Option<&[Option; 11]>, + observed_since: Option<&str>, (limit, adapter, has_event_id): (usize, &str, bool), ) -> Result { let mut columns: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES @@ -65,10 +66,17 @@ pub fn read_history_sql( )); } } - let filter = if greater.is_empty() { + let mut filters: Vec = Vec::new(); + if !greater.is_empty() { + filters.push(format!("({})", greater.join(" OR "))); + } + if let Some(since) = observed_since { + filters.push(format!("{OBSERVED_AT_COLUMN} >= {since}")); + } + let filter = if filters.is_empty() { String::new() } else { - format!(" WHERE {}", greater.join(" OR ")) + format!(" WHERE {}", filters.join(" AND ")) }; let order: Vec = columns .iter() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index df565a66a6..beaa693a20 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -82,7 +82,7 @@ def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_pr with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) - def page(after: list[str | None] | None = None) -> Any: + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -102,6 +102,7 @@ def page(after: list[str | None] | None = None) -> Any: "source_freshness": { "location": {"database": None, "schema": "main"}, "after": after, + "observed_since": observed_since, } } ) @@ -116,6 +117,11 @@ def page(after: list[str | None] | None = None) -> Any: old: dict[str, Any] = page() assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None + assert page(observed_since="2026-01-01T09:00:00Z") == old + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } for journal_mode in test_case.journal_modes: with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) @@ -287,6 +293,36 @@ def page(after: list[str | None] | None = None) -> Any: f"external-{index}" for index in range(104) } assert {row["target_database"] for row in imported} == {None, ""} + # The time bound is inclusive and independent of source-name/keyset ordering. + # SQL may read a superset within the boundary day, so a short decoded page + # must retain its continuation if the raw query filled the page. + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + zero: dict[str, Any] = page(observed_since="2026-02-01T00:00:00Z") + assert ( + len(zero["records"]) + len(page(zero["next_after"], "2026-02-01T00:00:00Z")["records"]) + == 105 + ) + assert sum(row["run_id"] == "external-0" for row in zero["records"]) == 2 + # An empty decoded page can still have another bounded SQL page to inspect. + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + assert page(observed_since="2030-01-01T00:00:00Z") == { + "records": [], + "next_after": None, + } assert not (project / ".sqlbuild/control.sqlite3").exists() (project / "warehouse.db").rename(project / "retained-warehouse.db") assert page() is None From 7c75072f15e831c586785fcab8ed3c45b242ee62 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:33:00 +0000 Subject: [PATCH 058/138] refactor: share warehouse history SQL construction Keep history row ordering and time-bound semantics unchanged while giving append-only state tables one keyset SQL owner. Rebuilt DuckDB and SQLite history regressions pass unchanged. --- .../native_host/warehouse_metadata.rs | 15 ++++++-- .../sqlbuild-state/src/freshness/constants.rs | 1 - .../sqlbuild-state/src/freshness/main/mod.rs | 1 - .../src/state_tables/constants.rs | 3 ++ .../src/state_tables/main/mod.rs | 1 + .../main/read_history_sql.rs | 37 +++++++++---------- crates/sqlbuild-state/src/state_tables/mod.rs | 1 + .../sqlbuild-state/src/state_tables/models.rs | 11 ++++++ 8 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 crates/sqlbuild-state/src/state_tables/constants.rs rename crates/sqlbuild-state/src/{freshness => state_tables}/main/read_history_sql.rs (81%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 3422a7d6f1..4958ffa439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -16,7 +16,8 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; -use sqlbuild_state::freshness::main::read_history_sql::read_history_sql; +use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; +use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; @@ -170,9 +171,15 @@ fn read_page( .transpose()?; let sql = read_history_sql( &qualified, - request.after.as_ref().map(|_| &literals), - observed_since.as_deref(), - (MAX_FRESHNESS_RECORDS, &adapter_name, has_event_id), + request.after.as_ref().map(|_| literals.as_slice()), + HistoryRead { + columns: &SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name), + timestamp_column: OBSERVED_AT_COLUMN, + observed_since: observed_since.as_deref(), + adapter: &adapter_name, + has_event_id, + limit: MAX_FRESHNESS_RECORDS, + }, ) .map_err(PyValueError::new_err)?; let result = call_with( diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 0dc528ac8c..184ed9e83b 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -57,6 +57,5 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; pub const OBSERVED_AT_COLUMN: &str = "observed_at"; pub const SQLITE_ADAPTER: &str = "sqlite"; -pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 5a1085a0e2..2e5f2dbe8f 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -3,7 +3,6 @@ pub mod configuration_error; pub mod data_version_hash; pub mod event_id; pub mod normalized_data_version; -pub mod read_history_sql; pub mod read_latest_sql; pub mod records_equivalent; pub mod relevant_sources_sql; diff --git a/crates/sqlbuild-state/src/state_tables/constants.rs b/crates/sqlbuild-state/src/state_tables/constants.rs new file mode 100644 index 0000000000..8bf6191df2 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/constants.rs @@ -0,0 +1,3 @@ +//! Adapter decisions shared by append-only state history reads. + +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; diff --git a/crates/sqlbuild-state/src/state_tables/main/mod.rs b/crates/sqlbuild-state/src/state_tables/main/mod.rs index 712abe33fc..6b46ba4e98 100644 --- a/crates/sqlbuild-state/src/state_tables/main/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/main/mod.rs @@ -1,2 +1,3 @@ pub mod create_table_sql; +pub mod read_history_sql; pub mod values_insert_sql; diff --git a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs similarity index 81% rename from crates/sqlbuild-state/src/freshness/main/read_history_sql.rs rename to crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs index e894646829..2d871377ba 100644 --- a/crates/sqlbuild-state/src/freshness/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -1,35 +1,34 @@ //! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. -use crate::freshness::constants::{ - OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, SQLSERVER_ADAPTER, -}; +use crate::state_tables::constants::SQLSERVER_ADAPTER; +use crate::state_tables::models::HistoryRead; /// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. pub fn read_history_sql( qualified_name: &str, - after: Option<&[Option; 11]>, - observed_since: Option<&str>, - (limit, adapter, has_event_id): (usize, &str, bool), + after: Option<&[Option]>, + read: HistoryRead<'_>, ) -> Result { - let mut columns: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES - .iter() - .map(|(name, _)| *name) - .collect(); + let HistoryRead { + columns, + timestamp_column, + observed_since, + adapter, + has_event_id, + limit, + } = read; + let mut columns = columns.to_vec(); let mut projection = columns.join(", "); projection.push_str(if has_event_id { ", event_id" } else { ", NULL AS event_id" }); - columns.push(if has_event_id { - SOURCE_FRESHNESS_EVENT_ID - } else { - "NULL" - }); + columns.push(if has_event_id { "event_id" } else { "NULL" }); let ordered = columns .iter() .map(|column| { - if *column == OBSERVED_AT_COLUMN { + if *column == timestamp_column { Ok((*column).to_owned()) } else { binary(column, adapter) @@ -43,7 +42,7 @@ pub fn read_history_sql( let literal = value .as_ref() .map(|value| { - if *column == OBSERVED_AT_COLUMN { + if *column == timestamp_column { Ok(value.clone()) } else { binary(value, adapter) @@ -71,7 +70,7 @@ pub fn read_history_sql( filters.push(format!("({})", greater.join(" OR "))); } if let Some(since) = observed_since { - filters.push(format!("{OBSERVED_AT_COLUMN} >= {since}")); + filters.push(format!("{timestamp_column} >= {since}")); } let filter = if filters.is_empty() { String::new() @@ -113,7 +112,7 @@ fn binary(value: &str, adapter: &str) -> Result { "clickhouse" => value.to_owned(), _ => { return Err(format!( - "source freshness history does not support adapter '{adapter}'" + "warehouse history does not support adapter '{adapter}'" )); } }) diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs index 1a70a48981..042268f15e 100644 --- a/crates/sqlbuild-state/src/state_tables/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -1,3 +1,4 @@ +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs index 7a343e134d..8c8a1f5792 100644 --- a/crates/sqlbuild-state/src/state_tables/models.rs +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -7,3 +7,14 @@ pub struct TableColumn { pub rendered_type: String, pub required: bool, } + +/// A bounded legacy-compatible history projection, with its optional source event ID. +#[derive(Clone, Copy, Debug)] +pub struct HistoryRead<'a> { + pub columns: &'a [&'a str], + pub timestamp_column: &'a str, + pub observed_since: Option<&'a str>, + pub adapter: &'a str, + pub has_event_id: bool, + pub limit: usize, +} From 7aee5010a9cb4fbb9ec7034d2d6ce2b94c512080 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:51:39 +0000 Subject: [PATCH 059/138] feat: read bounded fingerprint history from locations Read fingerprint observations through the isolated location protocol without changing latest readers or writing warehouse state. Preserve source IDs and predecessor rows, decode definitions through the existing reader, and keep raw keyset cursors independent of exact UTC bounds. Cover populated schemas, retries, mixed identities, Unicode storage and SQLite offsets across a date boundary. --- .../legacy_warehouse_fingerprint_offsets.sql | 10 + crates/README.md | 22 +- .../src/warehouse_metadata/constants.rs | 2 +- .../main/ingest_source_freshness.rs | 6 +- .../src/warehouse_metadata/models.rs | 52 ++++- .../_helpers/fingerprints/latest_rows.rs | 2 +- .../_helpers/location_command/runtime.rs | 14 +- .../native_host/warehouse_metadata.rs | 197 ++++++++++------ .../publication/_test_types.py | 12 + .../compiled_project/publication/helpers.py | 64 ++++++ .../publication/test_warehouse_metadata.py | 212 +++++++++++++----- 11 files changed, 452 insertions(+), 141 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql new file mode 100644 index 0000000000..fe7ba68f98 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql @@ -0,0 +1,10 @@ +-- SQLite predecessor text can carry the writer's offset, across the UTC date boundary. +-- Load after legacy_warehouse_state.sql, before any writer adds event_id. +INSERT INTO main._sqlbuild_fingerprints ( + node_type, node_name, target_database, target_schema, target_name, run_id, + definition_hash, version_hash, schema_fingerprint, definition_b64, metadata_json_b64, ts +) VALUES ( + 'model', 'offset_orders', NULL, 'main', 'offset_orders', 'offset-run', + 'legacy-definition', 'legacy-version', 'legacy-schema', + 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2025-12-31T23:00:00-11:00' +); diff --git a/crates/README.md b/crates/README.md index 03c551e8d2..e1727686ae 100644 --- a/crates/README.md +++ b/crates/README.md @@ -124,16 +124,20 @@ is normalized to UTC ISO text with six fractional digits and a `+00:00` offset. observation retains its ID; another run or observation is distinct. Definitions, display values, and future storage columns do not alter these identities. -The location history reader is read-only, including local database connections: it neither creates -databases nor evolves tables. Source-freshness ingestion preserves warehouse IDs, scoped by location, -target and physical state table. Only missing or NULL IDs use the fixed legacy observation tuple, -never a hash of the evolving full row. Bounded pages commit atomically under the current location -session and snapshot generation, and repeated pages are idempotent. A history read can hold an +The location history reader pages fingerprint and source-freshness observations without selecting +only the latest row. It is read-only, including local database connections: it neither creates +databases nor evolves tables. Fingerprint definitions and metadata use the existing storage decoder; +raw storage values remain the keyset cursor, separate from decoded UTC timestamps and text. +Source-freshness ingestion preserves warehouse IDs, scoped by location, target and physical state +table. Only missing or NULL IDs use the fixed legacy observation tuple, never a hash of the evolving +full row. Its bounded pages commit atomically under the current location session and snapshot +generation, and repeated pages are idempotent. A history read can hold an inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. -SQLite's SQL prefilter includes the boundary day so legacy space-separated timestamps and variable -fractional precision remain readable; decoded UTC timestamps enforce the exact bound. Other adapters -prefilter at whole-second precision. Continuation follows the raw row count: even an empty decoded -page can have more rows to inspect. These are read/ingestion primitives; automatic polling, +SQLite's SQL prefilter includes the boundary day and its predecessor so legacy separators, variable +fractional precision and writer offsets crossing a UTC date remain readable; decoded UTC timestamps +enforce the exact bound. Other adapters prefilter at whole-second precision. Continuation follows +the raw row count: even an empty decoded page can have more rows to inspect. Fingerprint ingestion +and per-hash definition storage remain separate from this read-only transport. Automatic polling, watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, not implemented by the page reader. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 05ee34c05a..2615174d34 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -1,3 +1,3 @@ //! Limits bound each authenticated warehouse observation transaction. -pub const MAX_FRESHNESS_RECORDS: usize = 100; +pub const MAX_HISTORY_RECORDS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index 8f170f2fce..a734b0440b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -11,7 +11,7 @@ use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::models::SourceFreshnessBatch; pub async fn ingest_source_freshness( @@ -40,12 +40,12 @@ pub async fn ingest_source_freshness( .is_some_and(|value| value.trim().is_empty()) }); if batch.generation < 1 - || batch.records.len() > MAX_FRESHNESS_RECORDS + || batch.records.len() > MAX_HISTORY_RECORDS || invalid_identity || invalid_source_id { return Err(ControlError::Invalid(format!( - "source freshness sync needs a positive generation, at most {MAX_FRESHNESS_RECORDS} records and nonempty state identities", + "source freshness sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", ))); } let transaction = require_instance( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 13ac7c9600..3a59cbc391 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -1,4 +1,4 @@ -//! Existing source-freshness state rows, with their physical state-table provenance. +//! Existing warehouse state rows, with their physical state-table provenance. use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -50,9 +50,53 @@ pub struct SourceFreshnessRead { pub observed_since: Option>, } +/// Fingerprints remain history here; latest-row selection belongs to the existing planner reader. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFingerprintRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: DateTime, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct SourceFreshnessPage { - pub records: Vec, - pub next_after: Option<[Option; 11]>, +pub struct FingerprintRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 13]>, + pub observed_since: Option>, +} + +/// The private interpreter accepts only these two known state tables, never arbitrary SQL. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseHistoryRead { + SourceFreshness(SourceFreshnessRead), + Fingerprints(FingerprintRead), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WarehouseHistoryRecord { + SourceFreshness(WarehouseFreshnessRecord), + Fingerprint(WarehouseFingerprintRecord), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseHistoryPage { + pub records: Vec, + pub next_after: Option>>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index e977d6a155..3347752271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -130,7 +130,7 @@ fn borrowed(identities: &[(String, String)]) -> Vec> { .collect() } -fn row_fingerprint<'py>( +pub(crate) fn row_fingerprint<'py>( fingerprint_class: &Bound<'py, PyAny>, from_iso: &Bound<'py, PyAny>, row: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 1ca8f40504..bc75c07fdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -6,13 +6,13 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; -use sqlbuild_orchestration::warehouse_metadata::models::SourceFreshnessRead; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryRead; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; -use crate::bindings::_helpers::native_host::warehouse_metadata::read_source_freshness; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_history; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; @@ -46,13 +46,15 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul } }; serde_json::to_string(&record) - } else if request.get("source_freshness").is_some() { - let metadata: SourceFreshnessRead = - serde_json::from_value(request["source_freshness"].clone()).map_err(|_| { + } else if request.get("source_freshness").is_some() + || request.get("fingerprints").is_some() + { + let metadata: WarehouseHistoryRead = + serde_json::from_value(request).map_err(|_| { PyValueError::new_err("runtime received an invalid metadata read") })?; let arguments = location_arguments(py, Some(options.clone()))?; - let page = match read_source_freshness(py, &arguments, &metadata) { + let page = match read_history(py, &arguments, &metadata) { Ok(page) => Some(page), Err(_) => { write_console( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 4958ffa439..d07490aeb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -4,14 +4,15 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use serde_json::{Value, json}; +use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGERPRINT_COLUMNS}; use sqlbuild_config::command_connection::constants::{ DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; -use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; use sqlbuild_core::models::SqlValue; -use sqlbuild_orchestration::warehouse_metadata::constants::MAX_FRESHNESS_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - SourceFreshnessPage, SourceFreshnessRead, WarehouseFreshnessRecord, + WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, }; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, @@ -22,18 +23,25 @@ use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::freshness::state_io::{qualified_table_name, row_record}; +use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; +use crate::bindings::_helpers::freshness::state_io::row_record; +use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::{construct::record_class, ids::RecordId}; -pub(crate) fn read_source_freshness( +pub(crate) fn read_history( py: Python<'_>, arguments: &KeywordArguments<'_>, - request: &SourceFreshnessRead, -) -> PyResult { - if request.location.schema.trim().is_empty() { + request: &WarehouseHistoryRead, +) -> PyResult { + let location = match request { + WarehouseHistoryRead::SourceFreshness(read) => &read.location, + WarehouseHistoryRead::Fingerprints(read) => &read.location, + }; + if location.schema.trim().is_empty() { return Err(PyValueError::new_err( "metadata state schema must not be blank", )); @@ -87,50 +95,66 @@ pub(crate) fn read_source_freshness( fn read_page( py: Python<'_>, (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - request: &SourceFreshnessRead, -) -> PyResult { - let database = request.location.database.as_ref().map_or_else( + request: &WarehouseHistoryRead, +) -> PyResult { + let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let (location, after, since, table, columns, timestamp) = match request { + WarehouseHistoryRead::SourceFreshness(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + SOURCE_FRESHNESS_TABLE_NAME, + freshness_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + FINGERPRINT_TABLE_NAME, + FINGERPRINT_COLUMNS.as_slice(), + COLUMN_TIMESTAMP, + ), + }; + let database = location.database.as_ref().map_or_else( || py.None().into_bound(py), |value| PyString::new(py, value).into_any(), ); - let schema = PyString::new(py, &request.location.schema).into_any(); - let name = PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(); + let schema = PyString::new(py, &location.schema).into_any(); + let name = PyString::new(py, table).into_any(); let exists = call_with( &adapter.getattr("relation_exists")?, &[ ("connection", connection.clone()), ("database", database.clone()), ("schema", schema.clone()), - ("name", name), + ("name", name.clone()), ], )? .is_truthy()?; if !exists { - return Ok(SourceFreshnessPage { + return Ok(WarehouseHistoryPage { records: Vec::new(), next_after: None, }); } - let columns = call_with( + let catalog_columns = call_with( &adapter.getattr("get_columns")?, &[ ("connection", connection.clone()), ("database", database.clone()), ("schema", schema.clone()), - ( - "name", - PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(), - ), + ("name", name.clone()), ], )?; let mut has_event_id = false; - for column in columns.try_iter()? { + for column in catalog_columns.try_iter()? { has_event_id |= field(&column?, "name")? .extract::()? .eq_ignore_ascii_case("event_id"); } - let mut literals: [Option; 11] = Default::default(); - if let Some(after) = &request.after { + let mut literals = vec![None; columns.len() + 1]; + if let Some(after) = after { let classes = SqlValueClasses::load(py)?; for (output, value) in literals.iter_mut().zip(after) { if let Some(value) = value { @@ -145,21 +169,24 @@ fn read_page( } } } - let qualified = qualified_table_name( + let qualified: String = call_with( &adapter.getattr("render_qualified_name")?, - (&database, &schema), - )?; + &[("database", database), ("schema", schema), ("name", name)], + )? + .extract()?; let adapter_name = adapter.getattr("adapter_name")?.extract::()?; - let observed_since = request - .observed_since + let observed_since = since .map(|since| { - let text = since - .format(if adapter_name == SQLITE_ADAPTER { - "%Y-%m-%d" - } else { - "%Y-%m-%dT%H:%M:%S+00:00" - }) - .to_string(); + let text = if adapter_name == SQLITE_ADAPTER { + since + .date_naive() + .pred_opt() + .unwrap_or(since.date_naive()) + .format("%Y-%m-%d") + .to_string() + } else { + since.format("%Y-%m-%dT%H:%M:%S+00:00").to_string() + }; let classes = SqlValueClasses::load(py)?; let value = sql_value(&SqlValue::String(text), &classes)?; call_with( @@ -171,14 +198,14 @@ fn read_page( .transpose()?; let sql = read_history_sql( &qualified, - request.after.as_ref().map(|_| literals.as_slice()), + after.map(|_| literals.as_slice()), HistoryRead { - columns: &SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name), - timestamp_column: OBSERVED_AT_COLUMN, + columns, + timestamp_column: timestamp, observed_since: observed_since.as_deref(), adapter: &adapter_name, has_event_id, - limit: MAX_FRESHNESS_RECORDS, + limit: MAX_HISTORY_RECORDS, }, ) .map_err(PyValueError::new_err)?; @@ -189,52 +216,94 @@ fn read_page( ("sql", PyString::new(py, &sql).into_any()), ( "limit", - pyo3::IntoPyObjectExt::into_bound_py_any(MAX_FRESHNESS_RECORDS, py)?, + pyo3::IntoPyObjectExt::into_bound_py_any(MAX_HISTORY_RECORDS, py)?, ), ], )?; - let mut records: Vec = Vec::new(); + let mut records: Vec = Vec::new(); let mut next_after = None; let mut read_count = 0; for row in field(&result, "rows")?.try_iter()? { read_count += 1; let row = row?; - let record = row_record(&row)?; - let mut cursor: [Option; 11] = Default::default(); - let mut value = json!({"state_database": request.location.database, "state_schema": request.location.schema}); - for (index, (name, _)) in SOURCE_FRESHNESS_COLUMN_TYPES.iter().enumerate() { + let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; + let mut cursor = vec![None; columns.len() + 1]; + for (index, value) in cursor.iter_mut().enumerate() { let raw = row.get_item(index)?; - cursor[index] = if raw.is_none() { + *value = if raw.is_none() { None } else { Some(raw.str()?.extract()?) }; - let item = field(&record, name)?; - value[*name] = if item.is_none() { - Value::Null - } else if *name == OBSERVED_AT_COLUMN { - json!(item.call_method0("isoformat")?.extract::()?) - } else { - json!(item.extract::()?) - }; } - cursor[10] = row.get_item(10)?.extract::>()?; - value["event_id"] = json!(cursor[10]); - let record = serde_json::from_value::(value) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - if request - .observed_since - .is_none_or(|since| record.observed_at >= since) - { + let observed = match &record { + WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, + WarehouseHistoryRecord::Fingerprint(record) => record.ts, + }; + if since.is_none_or(|since| observed >= since) { records.push(record); } next_after = Some(cursor); } - if read_count < MAX_FRESHNESS_RECORDS { + if read_count < MAX_HISTORY_RECORDS { next_after = None; } - Ok(SourceFreshnessPage { + Ok(WarehouseHistoryPage { records, next_after, }) } + +fn decode_record( + py: Python<'_>, + (request, row, qualified): (&WarehouseHistoryRead, &Bound<'_, PyAny>, &str), + (columns, timestamp): (&[&str], &str), +) -> PyResult { + let (location, record) = match request { + WarehouseHistoryRead::SourceFreshness(read) => (&read.location, row_record(row)?), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + row_fingerprint( + &record_class(py, RecordId::Fingerprint)?, + &py.import("datetime")? + .getattr("datetime")? + .getattr("fromisoformat")?, + row, + qualified, + )?, + ), + }; + let mut value = json!({ + "state_database": location.database, + "state_schema": location.schema, + "event_id": row.get_item(columns.len())?.extract::>()?, + }); + for name in columns { + let name = match *name { + "definition_b64" => "definition", + "metadata_json_b64" => "metadata_json", + name => name, + }; + let item = field(&record, name)?; + value[name] = if item.is_none() { + Value::Null + } else if name == timestamp { + json!( + presumed_utc_datetime(&item)? + .call_method0("isoformat")? + .extract::()? + ) + } else { + json!(item.extract::()?) + }; + } + match request { + WarehouseHistoryRead::SourceFreshness(_) => { + serde_json::from_value(value).map(WarehouseHistoryRecord::SourceFreshness) + } + WarehouseHistoryRead::Fingerprints(_) => { + serde_json::from_value(value).map(WarehouseHistoryRecord::Fingerprint) + } + } + .map_err(|error| PyValueError::new_err(error.to_string())) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 681b669497..20b540641f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -2,6 +2,7 @@ from collections.abc import Callable from dataclasses import dataclass, field +from datetime import datetime from pathlib import Path from subprocess import Popen from typing import Any, Literal, NamedTuple @@ -154,3 +155,14 @@ class WarehouseHistoryTestCase: connection_key: str journal_modes: tuple[str, ...] append_fingerprints: Callable[..., None] + + +@dataclass(frozen=True) +class FingerprintHistoryTestCase: + description: str + adapter_name: str + connection_key: str + observed_at: datetime + journal_modes: tuple[str, ...] + fixture_paths: tuple[Path, ...] + expected_legacy_runs: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 16170f6d72..ae1526b8c8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -26,6 +26,7 @@ import pytest from sqlbuild._native import Fingerprint, write_fingerprint +from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -66,6 +67,69 @@ def location_process( _ = process.wait(timeout=5) +def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) -> tuple[Any, Any]: + """The caller owns the returned real connection; declarations must never run during reads.""" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + f'name = "orders"\nadapter = "{adapter_name}"\n' + f'[connections.local]\n{connection_key} = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + }, + ) + adapter: Any = resolve_adapter(adapter_name=adapter_name) + connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) + legacy_sql: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" + ).read_text() + for statement in legacy_sql.rstrip().removesuffix(";").split(";"): + _ = adapter.execute(connection=connection, sql=statement) + return adapter, connection + + +def warehouse_history_page( + project: Path, + table: str, + after: list[str | None] | None = None, + observed_since: str | None = None, +) -> Any: + """Exercise the real isolated reader without a server, state database or declaration import.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "runtime", + "--server", + "http://127.0.0.1:1", + "--location-id", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + ], + input=json.dumps( + { + table: { + "location": {"database": None, "schema": "main"}, + "after": after, + "observed_since": observed_since, + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + def wait_for_generation( api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path ) -> dict[str, Any]: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index beaa693a20..2b328d4c5f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -2,10 +2,7 @@ import hashlib import json -import os import sqlite3 -import subprocess -import sys from contextlib import closing from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -19,17 +16,19 @@ SourceFreshnessRenderers, read_latest_fingerprints, read_latest_source_freshness, + write_fingerprints, write_source_freshness_records, ) -from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + FingerprintHistoryTestCase, WarehouseHistoryTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_contended_fingerprints, append_retried_fingerprints, + open_legacy_warehouse, + warehouse_history_page, ) -from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @pytest.mark.parametrize( @@ -59,61 +58,18 @@ def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_pr ) -> None: project: Path = tmp_path / "project" connection_key: str = test_case.connection_key - write_files( - project_dir=project, - files={ - "sqlbuild_project.toml": ( - f'name = "orders"\nadapter = "{test_case.adapter_name}"\n' - f'[connections.local]\n{connection_key} = "warehouse.db"\n' - '[targets.development]\nschema = "main"\n' - ), - "orchestration/orders.py": 'raise AssertionError("metadata must not compile user declarations")\n', - }, + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key ) - adapter: Any = resolve_adapter(adapter_name=test_case.adapter_name) - connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) - legacy_sql: str = Path( - "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" - ).read_text() - for statement in legacy_sql.rstrip().removesuffix(";").split(";"): - _ = adapter.execute(connection=connection, sql=statement) adapter.close(connection) for journal_mode in test_case.journal_modes: with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: - result: subprocess.CompletedProcess[str] = subprocess.run( - [ - str(Path(sys.executable).with_name("sqb")), - "--project-dir", - str(project), - "location", - "runtime", - "--server", - "http://127.0.0.1:1", - "--location-id", - "00000000-0000-0000-0000-000000000001", - "--target", - "development", - ], - input=json.dumps( - { - "source_freshness": { - "location": {"database": None, "schema": "main"}, - "after": after, - "observed_since": observed_since, - } - } - ) - + "\n", - text=True, - capture_output=True, - timeout=20, - env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, - ) - assert result.returncode == 0, result.stderr - return json.loads(result.stdout) + return warehouse_history_page(project, "source_freshness", after, observed_since) old: dict[str, Any] = page() assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None @@ -327,3 +283,153 @@ def page(after: list[str | None] | None = None, observed_since: str | None = Non (project / "warehouse.db").rename(project / "retained-warehouse.db") assert page() is None assert not (project / "warehouse.db").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintHistoryTestCase( + "DuckDB typed fingerprint timestamps", + "duckdb", + "database", + datetime(2026, 2, 1, tzinfo=UTC), + (), + (), + ("legacy-run",), + ), + FingerprintHistoryTestCase( + "SQLite offset fingerprint timestamps", + "sqlite", + "path", + datetime(2026, 1, 31, 23, tzinfo=timezone(timedelta(hours=-1))), + ("delete",), + ( + Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql" + ), + ), + ("legacy-run", "offset-run"), + ), + ], + ids=lambda case: case.description, +) +def test_given_fingerprint_history_when_read_in_isolated_pages_then_legacy_retries_definitions_and_time_bounds_survive( + test_case: FingerprintHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for fixture in test_case.fixture_paths: + _ = adapter.execute(connection=connection, sql=fixture.read_text()) + adapter.close(connection) + + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: + return warehouse_history_page(project, "fingerprints", after, observed_since) + + legacy: dict[str, Any] = page() + assert legacy["next_after"] is None + assert {row["run_id"] for row in legacy["records"]} == set(test_case.expected_legacy_runs) + assert legacy["records"][0]["event_id"] is None + assert legacy["records"][0]["definition"] == "SELECT 1 AS order_id" + assert legacy["records"][0]["metadata_json"] == "{}" + assert page(observed_since="2026-01-01T09:00:00Z") == legacy + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_fingerprints" + ) + } + fingerprints: tuple[Fingerprint, ...] = tuple( + Fingerprint( + node_type=("model", "seed")[index % 2], + node_name=("Orders", "orders", "orders ", "ordérs", "o'rders 🧾")[index % 5], + target_database=(None, "")[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + definition_hash="definition", + version_hash=f"version-{index}", + schema_fingerprint="schema", + definition="SELECT 'o''rders 🧾' AS order_note", + metadata_json='{"note":"orders 🧾"}', + ts=test_case.observed_at + timedelta(microseconds=index), + ) + for index in range(104) + ) + for batch in (fingerprints, fingerprints[:1]): + write_fingerprints( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprints=batch, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + assert len(pages[-1]["next_after"]) == 13 + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == ( + 100, + 5 + len(test_case.expected_legacy_runs), + ) + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + assert {row["run_id"] for row in imported} == set(test_case.expected_legacy_runs) | { + f"external-{index}" for index in range(104) + } + assert len({row["event_id"] for row in imported} - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == len(test_case.expected_legacy_runs) + assert {row["target_database"] for row in imported} == {None, ""} + first: list[dict[str, Any]] = list(filter(lambda row: row["run_id"] == "external-0", imported)) + assert len(first) == 2 and first[0] == first[1] + assert first[0]["definition"] == "SELECT 'o''rders 🧾' AS order_note" + assert first[0]["metadata_json"] == '{"note":"orders 🧾"}' + assert ( + first[0]["event_id"] + == hashlib.sha256( + b'["sqlbuild.fingerprint.event.v1","model","Orders",null,"main","orders","external-0","version-0"]' + ).hexdigest() + ) + assert datetime.fromisoformat(first[0]["ts"]) == datetime(2026, 2, 1, tzinfo=UTC) + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() From c51695c7132b9ef41ddd8abc90c40d4c282667a8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:30:52 +0000 Subject: [PATCH 060/138] fix: preserve fingerprint write metadata budgets Inspect the authoritative catalog and evolve predecessor schemas only after an insert fails, within the existing retry boundary. Keep successful single and batch writes at their existing statement and metadata budgets. Update nullable identity goldens and adapter test doubles. --- .../_helpers/fingerprints/storage_io.rs | 63 ++++++++++++------- .../snowflake/test_state_table_ddl.py | 2 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 9 +++ .../clone/_helpers/fingerprinting/helpers.py | 7 ++- .../sql_values/test_state_table_sql.py | 5 +- 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index b4612c1c7a..e78d3b228f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -98,11 +98,10 @@ fn create_sql( )) } -/// Create the table/indexes, then add the nullable identity column to a pre-existing table. +/// Create the table/indexes without an extra catalog read for a current or new table. fn setup<'py>( location: &Location<'_, 'py>, writer: &Writer<'_, 'py>, - request: &Request<'_, 'py>, describe: &dyn Fn(&PyErr) -> PyResult>, ) -> PyResult<()> { let py: Python<'_> = writer.execute.py(); @@ -124,28 +123,13 @@ fn setup<'py>( sqls.push(sql?.extract()?); } } - let database = location.database.into_pyobject(py)?.into_any(); - let schema = location.schema.into_pyobject(py)?.into_any(); retry_write( py, || { for sql in &sqls { execute_sql(writer, sql)?; } - ensure_event_id_column( - (writer.connection, writer.execute), - (&database, &schema), - (FINGERPRINT_TABLE_NAME, &qualified_table_name(location)?), - ( - &request - .optional("read_columns")? - .unwrap_or_else(|| py.None().into_bound(py)), - &request - .optional("render_add_columns")? - .unwrap_or_else(|| py.None().into_bound(py)), - writer.render_framework_type, - ), - ) + Ok(()) }, describe, ) @@ -158,6 +142,37 @@ fn execute_sql<'py>(writer: &Writer<'_, 'py>, sql: &str) -> PyResult( + location: &Location<'_, 'py>, + writer: &Writer<'_, 'py>, + request: &Request<'_, 'py>, + sql: &str, +) -> PyResult<()> { + let error = match execute_sql(writer, sql) { + Ok(_) => return Ok(()), + Err(error) => error, + }; + let py = writer.execute.py(); + if !error.is_instance_of::(py) || present(request, "read_columns")?.is_none() { + return Err(error); + } + ensure_event_id_column( + (writer.connection, writer.execute), + ( + &location.database.into_pyobject(py)?.into_any(), + &location.schema.into_pyobject(py)?.into_any(), + ), + (FINGERPRINT_TABLE_NAME, &qualified_table_name(location)?), + ( + &request.get("read_columns")?, + &request.get("render_add_columns")?, + writer.render_framework_type, + ), + )?; + execute_sql(writer, sql).map(|_| ()) +} + /// Log one failed attempt with `details` after the attempt counts, then back off. fn retry_pause( py: Python<'_>, @@ -406,8 +421,12 @@ fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let on_progress: Option> = present(&request, "on_progress")?; let parts: WriteParts<'py> = WriteParts::read(&request)?; let (location, writer) = (parts.location(), parts.writer()); - setup(&location, &writer, &request, &|error| { + setup(&location, &writer, &|error| { batch_warning(&fingerprints, error) })?; let batches = fingerprint_insert_batches( @@ -457,7 +476,7 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let members: Vec> = batch.iter().collect(); retry_write( py, - || execute_sql(&writer, &insert_sql).map(|_| ()), + || execute_insert(&location, &writer, &request, &insert_sql), &|error| batch_warning(&members, error), )?; completed += members.len(); diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py index 00496491bd..2991ef7135 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py @@ -121,7 +121,7 @@ def test_given_state_table_when_rendering_on_snowflake_then_permanent_with_90_da "target_schema VARCHAR, target_name VARCHAR, run_id VARCHAR NOT NULL, " "definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, " "schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, " - "metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL" + "metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL, event_id VARCHAR" ") DATA_RETENTION_TIME_IN_DAYS = 90" ), ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index e9aac9965c..d31b9ea6c7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -16,6 +16,7 @@ SourceFreshnessRecord, build_source_freshness_create_table_sql, ) +from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cost.models import CostResourceContext @@ -53,6 +54,14 @@ def render_qualified_name( def render_framework_type(self, framework_type: FrameworkType) -> str: return framework_type.value + def get_columns(self, **_scope: object) -> tuple[ColumnInfo, ...]: + return (ColumnInfo(name="event_id", type="VARCHAR"),) + + def render_add_columns( + self, *, destination: str, columns: tuple[ColumnInfo, ...] + ) -> tuple[str, ...]: + return DuckDbAdapter().render_add_columns(destination=destination, columns=columns) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: return build_source_freshness_create_table_sql( database=database, diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index bd12b224da..118a40d383 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -20,9 +20,10 @@ SeedPlanEntry, ) from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -class CloneFingerprintAdapter: +class CloneFingerprintAdapter(DuckDbAdapter): adapter_name: str = "test" def execute(self, connection: object, sql: str) -> object: @@ -59,8 +60,8 @@ def render_read_latest_fingerprints_sql(self, **kwargs: object) -> str: del kwargs return "select * from fingerprints" - def render_framework_type(self, value: object) -> str: - return str(value) + def render_framework_type(self, type_name: object) -> str: + return str(type_name) def render_create_fingerprint_table_sql(self, **kwargs: object) -> str: del kwargs diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py index 74e61e3e14..487e929215 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py +++ b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py @@ -27,9 +27,10 @@ expected_sql=( "INSERT INTO analytics._sqlbuild_source_freshness (source_name, " "target_database, target_schema, target_name, run_id, strategy, value_kind, " - "data_version, data_version_hash, observed_at) VALUES ('raw.o''rders', NULL, " + "data_version, data_version_hash, observed_at, event_id) VALUES ('raw.o''rders', NULL, " "'raw', 'orders', 'run-1', 'adapter', 'timestamp', " - "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00')" + "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00', " + "'a7a96057679375d1ba89ff2842c691f288d410dc1f502726feeb9cc0675bfd26')" ), ) ], From b352da7cc171ce684e77186afa175bcbfbf9fd11 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:09:24 +0000 Subject: [PATCH 061/138] feat: ingest warehouse fingerprints losslessly Mirror bounded fingerprint pages under the registered location session and published generation. Preserve warehouse event IDs and semantic hashes while sharing exact-content-addressed definition bodies with snapshot sources. Cover replay, legacy identities, atomic rollback and rebuilds on SQLite and PostgreSQL. --- .../fingerprint_definition_variants.json | 20 + crates/README.md | 25 +- .../src/api/_helpers/warehouse_metadata.rs | 20 +- .../src/api/main/serve_api.rs | 4 + .../sqlbuild-orchestration/src/api/models.rs | 3 +- .../src/platform/_helpers/work_client.rs | 21 +- .../src/snapshots/_helpers/content.rs | 17 +- .../src/snapshots/_helpers/preparation.rs | 3 +- .../src/snapshots/_helpers/publication.rs | 5 +- .../src/snapshots/_helpers/reading.rs | 2 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/compress.rs | 13 + .../src/store/main/decompress.rs | 16 + .../src/store/main/migrate_store.rs | 3 +- .../src/store/main/mod.rs | 3 + .../src/store/main/store_source_text.rs | 18 + .../src/store/warehouse_fingerprints.sql | 11 + .../_helpers/fingerprints.rs | 60 +++ .../warehouse_metadata/_helpers/ingestion.rs | 40 ++ .../src/warehouse_metadata/_helpers/mod.rs | 2 + .../main/ingest_fingerprints.rs | 75 ++++ .../main/ingest_source_freshness.rs | 37 +- .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 11 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_location_work.rs | 2 + .../tests/api/test_warehouse_metadata.rs | 7 + .../tests/api/warehouse_metadata/helpers.rs | 347 ++++++++++++++++++ .../tests/sensors/test_runs.rs | 4 + .../tests/snapshots/helpers.rs | 2 +- 31 files changed, 720 insertions(+), 56 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json create mode 100644 crates/sqlbuild-orchestration/src/store/main/compress.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/decompress.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/store_source_text.rs create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_fingerprints.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json new file mode 100644 index 0000000000..afcc0383c4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json @@ -0,0 +1,20 @@ +{ + "semantic_hash": "fd5b7a46925b590b562e8b32bc94ee204b05867e9c9c266cf5b734f1586ae6e0", + "definitions": [ + { + "sql": "select order_id from orders", + "text_hash": "76d5ce528df9a090f025926fca724f92ac23e1021941e5061d211fcb5ded1e77" + }, + { + "sql": "SELECT order_id FROM orders -- retained source comment\n", + "text_hash": "1daa5068b226bbc6b50d082920762afc31df73b0c02e9d7e9f05fe6b7297d205" + } + ], + "review_marker": { + "node_type": "review_schema", + "definition_hash": "", + "version_hash": "", + "schema_fingerprint": "", + "definition": "" + } +} diff --git a/crates/README.md b/crates/README.md index e1727686ae..7a8fe479b3 100644 --- a/crates/README.md +++ b/crates/README.md @@ -128,16 +128,29 @@ The location history reader pages fingerprint and source-freshness observations only the latest row. It is read-only, including local database connections: it neither creates databases nor evolves tables. Fingerprint definitions and metadata use the existing storage decoder; raw storage values remain the keyset cursor, separate from decoded UTC timestamps and text. -Source-freshness ingestion preserves warehouse IDs, scoped by location, target and physical state -table. Only missing or NULL IDs use the fixed legacy observation tuple, never a hash of the evolving -full row. Its bounded pages commit atomically under the current location session and snapshot -generation, and repeated pages are idempotent. A history read can hold an +Fingerprint and source-freshness ingestion preserve warehouse IDs, scoped by location, target and +physical state table. Only missing or NULL IDs use fixed legacy fields, never a hash of an evolving +full row. Bounded pages of at most 100 observations commit atomically under the current location +session and snapshot generation, and repeated pages are idempotent. Fingerprint legacy IDs are +SHA-256 over a compact JSON array: `"sqlbuild.fingerprint.legacy.v1"`, node kind/name, target +database/schema/name, run ID, definition hash, version hash, schema fingerprint, exact definition, +metadata JSON text and UTC observation time (six fractional digits, `+00:00`). + +Fingerprint events retain their warehouse `definition_hash` unchanged. This is a semantic token +hash: SQL with different whitespace, comments or keyword case can share it. The separate +`definition_text_hash` references the existing immutable `sqb_source_texts` store, keyed by SHA-256 +of the exact UTF-8 bytes and using its existing compressed encoding. Identical bytes share a body; +semantic-hash-equivalent SQL variants remain lossless. Event JSON omits the definition body, and +the event and its referenced body commit or roll back together. Rebuilding the mirror by replaying +history preserves source identities and definitions; no warehouse mutation is involved. + +A history read can hold an inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. SQLite's SQL prefilter includes the boundary day and its predecessor so legacy separators, variable fractional precision and writer offsets crossing a UTC date remain readable; decoded UTC timestamps enforce the exact bound. Other adapters prefilter at whole-second precision. Continuation follows -the raw row count: even an empty decoded page can have more rows to inspect. Fingerprint ingestion -and per-hash definition storage remain separate from this read-only transport. Automatic polling, +the raw row count: even an empty decoded page can have more rows to inspect. Ingestion and per-hash +definition storage are separate from this read-only transport. Automatic polling, watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, not implemented by the page reader. diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs index 5ddb1f2001..5cca1d650f 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs @@ -9,8 +9,9 @@ use crate::api::errors::ApiError; use crate::api::models::MutationResponse; use crate::credentials::models::TokenIdentity; use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::ingest_fingerprints::ingest_fingerprints; use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; -use crate::warehouse_metadata::models::SourceFreshnessBatch; +use crate::warehouse_metadata::models::{FingerprintBatch, SourceFreshnessBatch}; #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, @@ -28,3 +29,20 @@ pub(crate) async fn report( applied: ingest_source_freshness(&store, (location_id, &target), &batch).await?, })) } + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/fingerprints", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = FingerprintBatch, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report_fingerprints( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: ingest_fingerprints(&store, (location_id, &target), &batch).await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index dd000306a9..cd9bc03bb5 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -37,6 +37,10 @@ pub async fn serve_api( "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", post(warehouse_metadata::report), ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/fingerprints", + post(warehouse_metadata::report_fingerprints), + ) .route( "/api/locations/{location_id}/targets/{target}/snapshots", get(snapshots::current).post(snapshots::publish), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 9668db92c1..750c46b118 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -28,7 +28,8 @@ use crate::api::_helpers::{ trigger_reads::current, trigger_reads::events, trigger_reads::ticks, - warehouse_metadata::report + warehouse_metadata::report, + warehouse_metadata::report_fingerprints ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 5c1351ac68..09b423fee7 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -8,9 +8,28 @@ use crate::api::models::MutationResponse; use crate::platform::models::{ LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, }; -use crate::warehouse_metadata::models::SourceFreshnessBatch; +use crate::warehouse_metadata::models::{FingerprintBatch, SourceFreshnessBatch}; impl LocationWorkClient { + pub async fn sync_fingerprints( + &self, + target: &str, + batch: &FingerprintBatch, + ) -> Result { + let endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "targets", + target, + "metadata", + "fingerprints", + ])?; + let response: MutationResponse = + client_response(self.connection.client.post(endpoint).json(batch)).await?; + Ok(response.applied) + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs index 3ce8bbd7c3..e744c7dc8f 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -1,14 +1,13 @@ //! Canonical content addresses and compressed immutable bodies. use std::collections::BTreeMap; -use std::io::{Read, Write}; -use flate2::{Compression, read::GzDecoder, write::GzEncoder}; use serde_json::Value; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use crate::snapshots::models::{NodeContent, PreparedNode, SnapshotNode}; use crate::store::errors::ControlError; +use crate::store::main::compress::compress; fn canonical(value: &Value) -> Value { match value { @@ -28,20 +27,6 @@ pub(crate) fn json_bytes(value: &Value) -> Result, ControlError> { serde_json::to_vec(&canonical(value)).map_err(|error| ControlError::Invalid(error.to_string())) } -pub(crate) fn compress(bytes: &[u8]) -> Result, ControlError> { - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder.write_all(bytes)?; - Ok(encoder.finish()?) -} - -pub(crate) fn decompress(value: &Value) -> Result, ControlError> { - let bytes: Vec = serde_json::from_value(value.clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let mut output: Vec = Vec::new(); - GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; - Ok(output) -} - pub(crate) fn node_content(node: &SnapshotNode) -> Result, ControlError> { let mut sources: BTreeMap<&'static str, (String, &str)> = BTreeMap::new(); for (kind, text) in [ diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs index 310f723503..a697f16dd4 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -4,9 +4,10 @@ use std::collections::BTreeMap; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use crate::snapshots::_helpers::content::{compress, json_bytes, prepare_node}; +use crate::snapshots::_helpers::content::{json_bytes, prepare_node}; use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; use crate::store::errors::ControlError; +use crate::store::main::compress::compress; pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result { if input.target.trim().is_empty() diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs index e05f14da2d..ec1ce071f5 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use crate::snapshots::models::{PreparedNode, SnapshotChange}; use crate::store::errors::ControlError; use crate::store::main::query::query; +use crate::store::main::store_source_text::store_source_text; use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn store_nodes( @@ -22,9 +23,7 @@ pub(crate) async fn store_nodes( "INSERT INTO sqb_snapshot_nodes (snapshot_id, node_key, node_kind, node_hash) VALUES ($1, $2, $3, $4)", vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(node.kind), Parameter::Text(node.hash)]).await?; for (kind, (hash, body)) in node.sources { - (transaction, _) = query(transaction, - "INSERT INTO sqb_source_texts (text_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", - vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + transaction = store_source_text(transaction, (hash.clone(), body)).await?; (transaction, _) = query(transaction, "INSERT INTO sqb_snapshot_sources (snapshot_id, node_key, kind, text_hash) VALUES ($1, $2, $3, $4)", vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(kind.into()), Parameter::Text(hash)]).await?; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs index a350003e29..7f463dd024 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -6,9 +6,9 @@ use serde::de::DeserializeOwned; use serde_json::Value; use uuid::Uuid; -use crate::snapshots::_helpers::content::decompress; use crate::snapshots::models::{SnapshotNode, SnapshotRecord}; use crate::store::errors::ControlError; +use crate::store::main::decompress::decompress; use crate::store::main::{decode_row::decode_row, query::query}; use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 6199e096da..02c12874a0 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -15,6 +15,7 @@ pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshness.sql"); +pub(crate) const WAREHOUSE_FINGERPRINTS: &str = include_str!("../warehouse_fingerprints.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/compress.rs b/crates/sqlbuild-orchestration/src/store/main/compress.rs new file mode 100644 index 0000000000..d29c898e55 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/compress.rs @@ -0,0 +1,13 @@ +//! Shared encoding for immutable, content-addressed control-store bodies. + +use std::io::Write; + +use flate2::{Compression, write::GzEncoder}; + +use crate::store::errors::ControlError; + +pub(crate) fn compress(bytes: &[u8]) -> Result, ControlError> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/decompress.rs b/crates/sqlbuild-orchestration/src/store/main/decompress.rs new file mode 100644 index 0000000000..520cd4e7d7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/decompress.rs @@ -0,0 +1,16 @@ +//! Decode immutable compressed control-store bodies. + +use std::io::Read; + +use flate2::read::GzDecoder; +use serde_json::Value; + +use crate::store::errors::ControlError; + +pub(crate) fn decompress(value: &Value) -> Result, ControlError> { + let bytes: Vec = serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut output: Vec = Vec::new(); + GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; + Ok(output) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index f2237bd093..c3b4a99f16 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, - SCHEMA, SENSORS, WAREHOUSE_FRESHNESS, render, verify_migrations, + SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -37,6 +37,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("executed_nodes", EXECUTED_NODES), ("node_sensors", NODE_SENSORS), ("warehouse_freshness", WAREHOUSE_FRESHNESS), + ("warehouse_fingerprints", WAREHOUSE_FINGERPRINTS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index d8119ad010..e0d83e510a 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -2,10 +2,13 @@ pub(crate) mod append_control_event; pub(crate) mod append_recall; pub(crate) mod begin; pub(crate) mod commit; +pub(crate) mod compress; pub(crate) mod decode_row; +pub(crate) mod decompress; pub mod list_recall_invocations; pub mod migrate_store; pub mod open_store; pub(crate) mod query; pub mod read_recall_view; pub(crate) mod refresh_clock; +pub(crate) mod store_source_text; diff --git a/crates/sqlbuild-orchestration/src/store/main/store_source_text.rs b/crates/sqlbuild-orchestration/src/store/main/store_source_text.rs new file mode 100644 index 0000000000..3ea443c604 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/store_source_text.rs @@ -0,0 +1,18 @@ +//! Snapshot sources and warehouse definitions share exact-content-addressed immutable bodies. + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn store_source_text( + transaction: ControlTransaction, + (hash, body): (String, Vec), +) -> Result { + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_source_texts (text_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash), Parameter::Bytes(body)], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_fingerprints.sql b/crates/sqlbuild-orchestration/src/store/warehouse_fingerprints.sql new file mode 100644 index 0000000000..3d06595e40 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_fingerprints.sql @@ -0,0 +1,11 @@ +CREATE TABLE sqb_warehouse_fingerprints ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + definition_text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs new file mode 100644 index 0000000000..860d810f29 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs @@ -0,0 +1,60 @@ +//! Fixed legacy fingerprint keys and validation of bounded source history pages. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::errors::ControlError; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{FingerprintBatch, WarehouseFingerprintRecord}; + +pub(crate) fn validate_batch(batch: &FingerprintBatch) -> Result<(), ControlError> { + let invalid = batch + .records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.node_type, + &record.node_name, + &record.run_id, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_id = batch + .records + .iter() + .filter_map(|record| record.event_id.as_deref()) + .any(|value| value.trim().is_empty()); + if batch.generation < 1 || batch.records.len() > MAX_HISTORY_RECORDS || invalid || invalid_id { + return Err(ControlError::Invalid(format!( + "fingerprint sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", + ))); + } + Ok(()) +} + +/// Hash fixed predecessor fields only when the warehouse has no persisted identity. +pub(crate) fn source_event_id(record: &WarehouseFingerprintRecord) -> String { + if let Some(event_id) = &record.event_id { + return event_id.clone(); + } + sha256_hex( + json!([ + "sqlbuild.fingerprint.legacy.v1", + record.node_type, + record.node_name, + record.target_database, + record.target_schema, + record.target_name, + record.run_id, + record.definition_hash, + record.version_hash, + record.schema_fingerprint, + record.definition, + record.metadata_json, + record.ts.format("%Y-%m-%dT%H:%M:%S%.6f+00:00").to_string(), + ]) + .to_string() + .as_bytes(), + ) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs new file mode 100644 index 0000000000..4d0f200fea --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -0,0 +1,40 @@ +//! A metadata page shares one registered session, snapshot fence and physical identity scope. + +use uuid::Uuid; + +use crate::platform::main::require_instance::require_instance; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, query::query}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +pub(crate) async fn begin_ingestion( + store: &ControlStore, + (location_id, target): (Uuid, &str), + (session_id, generation): (Uuid, i64), +) -> Result { + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, session_id, target), + ) + .await?; + let (transaction, rows) = query( + transaction, + "SELECT s.snapshot_id FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2 AND s.generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(generation)], + ).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + Ok(transaction) +} + +pub(crate) fn scoped_event_id( + (location_id, target): (Uuid, &str), + (database, schema, table): (Option<&str>, &str, &str), + source_event_id: &str, +) -> Result { + let identity = serde_json::to_vec(&(target, database, schema, table, source_event_id)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok(Uuid::new_v5(&location_id, &identity)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs new file mode 100644 index 0000000000..c4dcbb340a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod fingerprints; +pub(crate) mod ingestion; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs new file mode 100644 index 0000000000..a2a9a4e44c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs @@ -0,0 +1,75 @@ +//! Import bounded fingerprint history without conflating semantic and exact-content hashes. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{ + commit::commit, compress::compress, query::query, store_source_text::store_source_text, +}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::fingerprints::{source_event_id, validate_batch}; +use crate::warehouse_metadata::_helpers::ingestion::{begin_ingestion, scoped_event_id}; +use crate::warehouse_metadata::models::FingerprintBatch; + +pub async fn ingest_fingerprints( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &FingerprintBatch, +) -> Result { + validate_batch(batch)?; + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await?; + let mut applied = false; + for record in &batch.records { + let source_event_id = source_event_id(record); + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_fingerprints", + ), + &source_event_id, + )?; + let existing; + (transaction, existing) = query( + transaction, + "SELECT event_id FROM sqb_warehouse_fingerprints WHERE event_id=$1", + vec![Parameter::Id(event_id)], + ) + .await?; + if !existing.is_empty() { + continue; + } + let definition_text_hash = sha256_hex(record.definition.as_bytes()); + let mut record_json = json!(record); + if let serde_json::Value::Object(object) = &mut record_json { + object.remove("definition"); + } + let received_at = transaction.now; + transaction = store_source_text( + transaction, + ( + definition_text_hash.clone(), + compress(record.definition.as_bytes())?, + ), + ) + .await?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), + Parameter::Json(record_json), Parameter::Time(received_at)]).await?; + applied |= !inserted.is_empty(); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index a734b0440b..9a7f93572b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -5,12 +5,11 @@ use sqlbuild_state::freshness::main::event_id::event_id as legacy_event_id; use sqlbuild_state::freshness::models::FreshnessEventIdentity; use uuid::Uuid; -use crate::platform::main::require_instance::require_instance; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{begin_ingestion, scoped_event_id}; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::models::SourceFreshnessBatch; @@ -48,20 +47,12 @@ pub async fn ingest_source_freshness( "source freshness sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", ))); } - let transaction = require_instance( - begin(store, TransactionMode::Write).await?, - (location_id, batch.session_id, target), + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), ) .await?; - let (mut transaction, rows) = query( - transaction, - "SELECT s.snapshot_id FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ - WHERE c.location_id=$1 AND c.target=$2 AND s.generation=$3", - vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(batch.generation)], - ).await?; - if rows.is_empty() { - return Err(ControlError::Fenced); - } let mut applied = false; for record in &batch.records { let source_event_id = record.event_id.clone().unwrap_or_else(|| { @@ -80,15 +71,15 @@ pub async fn ingest_source_freshness( .to_string(), }) }); - let identity = serde_json::to_vec(&( - target, - &record.state_database, - &record.state_schema, - "_sqlbuild_source_freshness", + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_source_freshness", + ), &source_event_id, - )) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let event_id = Uuid::new_v5(&location_id, &identity); + )?; let received_at = transaction.now; let inserted; (transaction, inserted) = query(transaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 0ff12b7aff..fc32896be6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1 +1,2 @@ +pub mod ingest_fingerprints; pub mod ingest_source_freshness; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs index 69fb999ac1..de8b740018 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs @@ -1,5 +1,6 @@ //! Warehouse-owned observations mirrored by authenticated code locations, never execution state. +pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 3a59cbc391..cb926f1a92 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -51,7 +51,7 @@ pub struct SourceFreshnessRead { } /// Fingerprints remain history here; latest-row selection belongs to the existing planner reader. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct WarehouseFingerprintRecord { pub event_id: Option, @@ -71,6 +71,15 @@ pub struct WarehouseFingerprintRecord { pub ts: DateTime, } +/// Fingerprint observations from one registered reader, independent of automatic polling. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct FingerprintBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct FingerprintRead { diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 2faba15f57..bffc493c5c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -20,5 +20,7 @@ mod test_types; mod test_warehouse_metadata; #[path = "test_worker.rs"] mod test_worker; +#[path = "warehouse_metadata/helpers.rs"] +mod warehouse_metadata_helpers; #[path = "work/helpers.rs"] mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index a89d419bcd..0e84bbc2c1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,9 +46,11 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index efc31663b5..5869626d42 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -299,6 +299,13 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit serde_json::to_value(rows).expect("rows"), json!([{"target": "development", "n": 6}, {"target": "preview", "n": 4}]) ); + crate::api::tests::warehouse_metadata_helpers::fingerprint_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs new file mode 100644 index 0000000000..0e0ddc0487 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -0,0 +1,347 @@ +//! Exact definitions survive semantic-hash reuse, page retries and mirror rebuilds. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::LocationWorkClient; +use crate::store::main::decompress::decompress; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::models::FingerprintBatch; + +pub(crate) async fn fingerprint_history( + store: &ControlStore, + outbound: &LocationWorkClient, + (session_id, preview_session_id): (Uuid, Uuid), + (client, freshness_url, bearer): (&Client, &str, &str), +) { + // Upgrade the immediate predecessor while published snapshots and freshness facts exist. + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("upgrade"), + "DROP TABLE sqb_warehouse_fingerprints", + vec![], + ) + .await + .expect("predecessor table set"); + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_fingerprints'", + vec![], + ) + .await + .expect("predecessor schema version"); + commit(transaction).await.expect("populated predecessor"); + migrate_store(store).await.expect("additive mirror upgrade"); + let url = freshness_url.replace("source-freshness", "fingerprints"); + let variants_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json", + ); + let variants: Value = + serde_json::from_str(&std::fs::read_to_string(variants_path).expect("DuckShop fixture")) + .expect("DuckShop semantic-equivalence case"); + let definition = variants["definitions"][0]["sql"].as_str().expect("SQL"); + let variant = variants["definitions"][1]["sql"] + .as_str() + .expect("variant SQL"); + // Independently established with the public native hash function; raw UTF-8 differs. + let semantic = variants["semantic_hash"].as_str().expect("semantic hash"); + let record = json!({ + "event_id": "warehouse-fingerprint-1", "state_database": null, "state_schema": "main", + "node_type": "model", "node_name": "orders", "target_database": null, + "target_schema": "main", "target_name": "orders", "run_id": "external-command-1", + "definition_hash": semantic, "version_hash": "version-1", "schema_fingerprint": "schema-1", + "definition": definition, "metadata_json": "{\"audit\":1}", "ts": "2026-01-01T10:00:00Z", + }); + let mut other = record.clone(); + other["event_id"] = json!("warehouse-fingerprint-2"); + other["run_id"] = json!("external-command-2"); + other["definition"] = json!(variant); + let mut legacy = record.clone(); + legacy.as_object_mut().expect("record").remove("event_id"); + legacy["run_id"] = json!("legacy-command"); + let mut legacy_null = legacy.clone(); + legacy_null["event_id"] = Value::Null; + legacy_null["ts"] = json!("2026-01-01T11:00:00+01:00"); + let mut page = vec![record.clone(); 100]; + page[1] = other; + page[2] = legacy; + page[3] = legacy_null.clone(); + legacy_null["ts"] = json!("2026-01-01T10:00:00.000001Z"); + page[4] = legacy_null; + let batch = json!({"session_id": session_id, "generation": 1, "records": page}); + let typed: FingerprintBatch = serde_json::from_value(batch.clone()).expect("typed page"); + for expected in [true, false] { + assert_eq!( + outbound + .sync_fingerprints("development", &typed) + .await + .expect("sync"), + expected + ); + } + let before = mirrored_fingerprints(store).await; + assert_eq!( + before.len(), + 4, + "replay and absent/NULL legacy IDs deduplicate" + ); + let mut texts = Vec::new(); + let mut hashes = std::collections::BTreeSet::new(); + let mut source_ids = std::collections::BTreeSet::new(); + for row in &before { + assert_eq!(row["record_json"]["definition_hash"], semantic); + assert!( + row["record_json"].get("definition").is_none(), + "event holds a reference, not another body" + ); + hashes.insert(row["definition_text_hash"].as_str().expect("hash")); + source_ids.insert(row["source_event_id"].as_str().expect("source ID")); + texts + .push(String::from_utf8(decompress(&row["body"]).expect("decompress")).expect("UTF-8")); + } + texts.sort(); + assert_eq!(texts, [variant, definition, definition, definition]); + assert_eq!( + source_ids, + std::collections::BTreeSet::from([ + "warehouse-fingerprint-1", + "warehouse-fingerprint-2", + "39e801c87fb3856345ca8ff4b4581edcd4439d8688c5fb399ac331948a004287", + "ac87eafefb99f262e0304fb1f5ec891a820af7ff9cdf1e97f44306861ffbbb18", + ]) + ); + assert_eq!( + hashes, + std::collections::BTreeSet::from([ + variants["definitions"][0]["text_hash"] + .as_str() + .expect("hash"), + variants["definitions"][1]["text_hash"] + .as_str() + .expect("hash"), + ]) + ); + let mut stale = batch.clone(); + stale["generation"] = json!(2); + let mut oversized = batch.clone(); + oversized["records"] = json!(vec![record.clone(); 101]); + let mut blank = batch.clone(); + blank["records"][0]["event_id"] = json!(" "); + let mut foreign = batch.clone(); + foreign["session_id"] = json!(Uuid::now_v7()); + for (endpoint, body, status) in [ + (url.clone(), stale, StatusCode::CONFLICT), + (url.clone(), oversized, StatusCode::BAD_REQUEST), + (url.clone(), blank, StatusCode::BAD_REQUEST), + (url.clone(), foreign, StatusCode::BAD_REQUEST), + ( + url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string(), + ), + batch.clone(), + StatusCode::FORBIDDEN, + ), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(bearer) + .json(&body) + .send() + .await + .expect("rejected page") + .status(), + status + ); + } + assert_eq!( + client + .post(&url) + .json(&batch) + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "fingerprint event and definition page roll back together", + table: "sqb_warehouse_fingerprints", + operation: "INSERT", + condition: "WHEN NEW.record_json->>'node_name'='inventory'", + sqlite_body: "SELECT RAISE(ABORT, 'injected fingerprint failure')", + postgres_body: "RAISE EXCEPTION 'injected fingerprint failure'", + }; + let mut first = record.clone(); + first["event_id"] = json!("new-fingerprint-1"); + first["definition"] = json!("select product_id from products"); + let mut second = first.clone(); + second["event_id"] = json!("new-fingerprint-2"); + second["node_name"] = json!("inventory"); + second["definition"] = json!("select inventory_id from inventory"); + let additional = json!({"session_id": session_id, "generation": 1, "records": [first, second]}); + let bodies_before = source_bodies(store).await; + install_fault(store, &fault).await; + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("faulted page") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + assert_eq!( + mirrored_fingerprints(store).await, + before, + "no partial events" + ); + assert_eq!( + source_bodies(store).await, + bodies_before, + "no orphan definitions" + ); + // Rebuild from an empty mirror, including its unshared exact-content bodies. + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("write"), + "DELETE FROM sqb_warehouse_fingerprints", + vec![], + ) + .await + .expect("wipe mirror"); + let (transaction, _) = query(transaction, + "DELETE FROM sqb_source_texts WHERE NOT EXISTS (SELECT 1 FROM sqb_snapshot_sources s WHERE s.text_hash=sqb_source_texts.text_hash)", + vec![]).await.expect("wipe unshared bodies"); + commit(transaction).await.expect("commit rebuild start"); + assert!( + outbound + .sync_fingerprints("development", &typed) + .await + .expect("resync") + ); + assert_eq!( + mirrored_fingerprints(store).await, + before, + "same identities, facts and exact SQL after resync" + ); + assert_eq!( + source_bodies(store).await, + bodies_before, + "same deduplicated body set" + ); + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("retry faulted page") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + // Review ownership markers are supported warehouse rows with intentionally empty hashes/SQL. + let mut marker = record.clone(); + marker["event_id"] = json!("warehouse-review-marker"); + for (key, value) in variants["review_marker"] + .as_object() + .expect("marker") + .iter() + { + marker[key] = value.clone(); + } + let marker: FingerprintBatch = serde_json::from_value(json!({ + "session_id": session_id, "generation": 1, "records": [marker], + })) + .expect("review marker page"); + assert!( + outbound + .sync_fingerprints("development", &marker) + .await + .expect("review marker sync") + ); + let rows = mirrored_fingerprints(store).await; + let marker = rows + .iter() + .find(|row| row["source_event_id"] == "warehouse-review-marker") + .expect("stored marker"); + assert_eq!(marker["record_json"]["definition_hash"], ""); + assert_eq!(decompress(&marker["body"]).expect("marker definition"), b""); + let bodies = source_bodies(store).await; + let mut scoped = record.clone(); + scoped["state_schema"] = json!("reporting"); + let mut other_database = record.clone(); + other_database["state_database"] = json!("analytics"); + let scoped: FingerprintBatch = serde_json::from_value(json!({ + "session_id": session_id, "generation": 1, "records": [scoped, other_database], + })) + .expect("physical scopes"); + assert!( + outbound + .sync_fingerprints("development", &scoped) + .await + .expect("distinct physical scopes") + ); + let preview: FingerprintBatch = serde_json::from_value(json!({ + "session_id": preview_session_id, "generation": 1, "records": [record], + })) + .expect("preview target"); + assert!( + outbound + .sync_fingerprints("preview", &preview) + .await + .expect("distinct target scope") + ); + let after = mirrored_fingerprints(store).await; + assert_eq!( + after.len(), + 10, + "identical source IDs in other physical/target scopes are independent" + ); + assert_eq!( + source_bodies(store).await, + bodies, + "scopes share identical definition bytes" + ); + migrate_store(store).await.expect("warm migration"); + assert_eq!( + mirrored_fingerprints(store).await, + after, + "warm migration preserves imported history" + ); +} + +async fn mirrored_fingerprints(store: &ControlStore) -> Vec { + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + "SELECT f.event_id,f.source_event_id,f.definition_text_hash,f.record_json,t.body FROM sqb_warehouse_fingerprints f \ + JOIN sqb_source_texts t ON t.text_hash=f.definition_text_hash ORDER BY f.event_id", vec![]).await.expect("mirrored definitions"); + commit(transaction).await.expect("close read"); + rows.into_iter() + .map(|row| serde_json::to_value(row).expect("row")) + .collect() +} + +async fn source_bodies(store: &ControlStore) -> Vec { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT text_hash,body FROM sqb_source_texts ORDER BY text_hash", + vec![], + ) + .await + .expect("immutable bodies"); + commit(transaction).await.expect("close read"); + rows.into_iter() + .map(|row| serde_json::to_value(row).expect("row")) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 44a6f6f588..1d3b9b0cf5 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,10 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP TABLE sqb_warehouse_fingerprints", vec![]) + .await + .expect("remove subsequent fingerprint mirror migration"); let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_freshness", vec![]) .await .expect("remove subsequent warehouse mirror migration"); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 2badd5ab9b..86546af384 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_warehouse_freshness; \ + "DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ From 30666092ac1f14ecde3f0847870296561227ff32 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:50:18 +0000 Subject: [PATCH 062/138] feat: stream scoped changes and supervise local development Publish scoped invalidations only after authoritative commits, recover PostgreSQL listeners and retain SQLite publication through caller cancellation. Share production roles in the loopback-only development supervisor with retained identity, paused trigger defaults, isolated reloads and graceful shutdown. Exclude the local control database and sidecars from source watching without forwarding PostgreSQL credentials. Resolve local-only targets through the compiler precedence contract. Cover real DuckDB runs, SSE edits, restart identity and transactional SQLite/PostgreSQL failures. --- Cargo.lock | 1 + Cargo.toml | 1 + .../orchestration/development_orders.py | 8 + crates/README.md | 41 ++ .../sqlbuild-cli/src/argv/command_table.json | 22 + .../src/native_commands/_helpers/execution.rs | 135 ++++++ .../src/native_commands/_helpers/paths.rs | 9 +- .../native_commands/_helpers/project_facts.rs | 43 ++ .../src/native_commands/main/_run_control.rs | 1 + .../src/native_commands/main/_run_dev.rs | 82 ++++ .../src/native_commands/main/mod.rs | 1 + .../main/run_location_invocation.rs | 31 +- .../src/workers/_helpers/child.rs | 5 +- .../src/workers/_helpers/supervisor.rs | 19 +- crates/sqlbuild-cli/src/workers/main/mod.rs | 2 + .../workers/main/open_development_workers.rs | 33 ++ .../src/workers/main/terminate_child.rs | 21 + crates/sqlbuild-orchestration/Cargo.toml | 1 + .../src/api/_helpers/authentication.rs | 79 +++- .../src/api/_helpers/changes.rs | 178 ++++++++ .../src/api/_helpers/location_health.rs | 4 +- .../src/api/_helpers/mod.rs | 1 + .../src/api/_helpers/run_reads.rs | 4 +- .../src/api/_helpers/runs.rs | 2 +- .../src/api/_helpers/snapshots.rs | 24 +- .../src/api/_helpers/trigger_reads.rs | 2 +- .../src/api/_helpers/warehouse_metadata.rs | 2 +- .../src/api/main/serve_api.rs | 61 ++- .../sqlbuild-orchestration/src/api/models.rs | 38 +- .../src/credentials/_helpers/locations.rs | 37 ++ .../src/credentials/_helpers/mod.rs | 1 + .../src/credentials/main/create_location.rs | 39 +- .../credentials/main/development_location.rs | 53 +++ .../src/credentials/main/mod.rs | 1 + .../src/credentials/main/revoke_token.rs | 25 +- .../src/declarations/_helpers/development.rs | 63 +++ .../src/declarations/_helpers/mod.rs | 3 + .../declarations/_helpers/operator_change.rs | 69 +++ .../declarations/_helpers/operator_state.rs | 7 +- .../src/declarations/_helpers/publication.rs | 105 +++++ .../src/declarations/main/change_trigger.rs | 77 +--- .../src/declarations/main/mod.rs | 1 + .../main/pause_development_triggers.rs | 18 + .../declarations/main/publish_definitions.rs | 114 +---- .../src/runs/_helpers/lifecycle/events.rs | 7 +- .../src/runs/main/publish_run_output.rs | 5 +- .../src/runs/main/publish_run_recall.rs | 7 +- .../src/scheduler/_helpers/persistence.rs | 7 +- .../src/sensors/_helpers/facts.rs | 7 +- .../src/snapshots/main/_publish_snapshot.rs | 97 ++++ .../src/snapshots/main/mod.rs | 2 + .../main/publish_development_snapshot.rs | 13 + .../src/snapshots/main/publish_snapshot.rs | 89 +--- .../src/store/_helpers/changes.rs | 87 ++++ .../src/store/_helpers/migration.rs | 1 + .../src/store/_helpers/mod.rs | 1 + .../src/store/development_locations.sql | 6 + .../src/store/main/begin.rs | 47 +- .../src/store/main/commit.rs | 29 +- .../src/store/main/listen_changes.rs | 15 + .../src/store/main/migrate_store.rs | 6 +- .../src/store/main/mod.rs | 2 + .../src/store/main/subscribe_changes.rs | 33 ++ .../src/store/models.rs | 42 ++ .../main/ingest_fingerprints.rs | 7 +- .../main/ingest_source_freshness.rs | 7 +- .../tests/api/changes/helpers.rs | 172 +++++++ .../tests/api/helpers.rs | 10 +- .../tests/api/test_changes.rs | 419 ++++++++++++++++++ .../tests/api/test_contracts.rs | 6 + .../tests/api/test_development.rs | 207 +++++++++ .../tests/api/test_location_work.rs | 2 + .../tests/api/test_types.rs | 24 + .../tests/api/test_warehouse_metadata.rs | 26 +- .../tests/api/warehouse_metadata/helpers.rs | 9 +- .../tests/credentials/test_contracts.rs | 2 +- .../tests/scheduler/test_contracts.rs | 2 +- .../tests/scheduler/ticks/test_ticks.rs | 2 +- .../tests/sensors/test_runs.rs | 4 + .../tests/snapshots/helpers.rs | 2 +- .../tests/store/recall/test_contract.rs | 93 ++++ .../tests/store/recall/test_types.rs | 5 + .../publication/_test_types.py | 10 + .../compiled_project/publication/helpers.py | 11 +- .../compiled_project/publication/test_dev.py | 204 +++++++++ 85 files changed, 2790 insertions(+), 401 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/development_orders.py create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs create mode 100644 crates/sqlbuild-cli/src/workers/main/open_development_workers.rs create mode 100644 crates/sqlbuild-cli/src/workers/main/terminate_child.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/changes.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/development_location.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/changes.rs create mode 100644 crates/sqlbuild-orchestration/src/store/development_locations.sql create mode 100644 crates/sqlbuild-orchestration/src/store/main/listen_changes.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/changes/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_changes.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_development.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py diff --git a/Cargo.lock b/Cargo.lock index 37cab0cc6c..03070db69c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2589,6 +2589,7 @@ dependencies = [ "chrono-tz", "croner", "flate2", + "futures-util", "getrandom 0.4.3", "proptest", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index fd65e8af7b..57d58e503d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ rust-version = "1.88" [workspace.dependencies] axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } fensu-policy = "=0.22.0" +futures-util = { version = "0.3", default-features = false, features = ["std"] } # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" globset = "0.4" diff --git a/benchmarks/duckshop/edge_cases/orchestration/development_orders.py b/benchmarks/duckshop/edge_cases/orchestration/development_orders.py new file mode 100644 index 0000000000..7306a2147c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/development_orders.py @@ -0,0 +1,8 @@ +"""Development defaults pause code-enabled triggers without changing their definitions.""" + +from sqlbuild.orchestration import job, schedule + +annual = schedule(cron="0 0 1 1 *", timezone="UTC", starts_at="2099-01-01T00:00:00Z") +annual_orders = job( + "annual_orders", description="Build annual orders.", select="+orders", schedule=annual +) diff --git a/crates/README.md b/crates/README.md index 7a8fe479b3..7d4b6787f2 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,47 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Control-plane live updates + +`GET /api/locations/{location_id}/targets/{target}/changes` streams authenticated SSE +`changed` events containing only a `topic`. Location credentials retain their existing scope; +subscribers never receive another location's or target's invalidations. Topics are `runs`, +`run:`, `ticks`, `snapshot:` and `node_status:`. Clients refetch the +corresponding REST resource rather than deriving application state from a notification. + +Invalidations are deduplicated within the writing transaction. PostgreSQL publishes with +transactional `NOTIFY`, with each API replica holding a schema-isolated `LISTEN`; SQLite publishes +through an in-process broadcast immediately after successful COMMIT, even if the awaiting async +caller was cancelled. Rejected transactions publish nothing. Snapshot, run, trigger, sensor and +warehouse-ingestion owners mark their own changes without adding a second durable event log. + +The first event, subscriber lag and restored PostgreSQL subscriptions emit `all`, requiring a +complete authorized REST refetch. There is no durable SSE cursor or replay guarantee. Recoverable +listener failures retry without stopping HTTP serving; fatal configuration/protocol failures remain +errors. Every event and 15-second keepalive revalidates the credential against the store. Revocation +invalidates active streams across replicas; expired or inactive credentials disclose no subsequent +changes. Server shutdown closes live streams instead of waiting indefinitely for their bodies. + +## Local development supervisor + +`sqb dev` runs the native API, scheduler and reloadable code location together. Its default +store is `.sqlbuild/control.sqlite3` in the canonical project directory. `--control-store-url` +selects another control store, `--target` overrides the configured target, `--port 0` chooses a +free port, and `--no-scheduler` leaves publication and reads active without launching runs. +This backend-only command does not serve a UI. + +Authentication is disabled only for the explicit development listener on loopback. Host and Origin +checks reject browser cross-origin and DNS-rebinding requests; ordinary `sqb server` authentication +is unchanged. Development retains an opaque location identity per canonical project and target. +The spawned code location receives no control-store credentials and uses the ordinary HTTP +publication, reload and health paths; workers retain their ordinary run-scoped credentials. + +Triggers start paused before scheduling can begin. Publication pauses newly introduced or revived +triggers in the same transaction as the snapshot, using ordinary audited operator facts rather +than rewriting code definitions. Reload preserves explicit operator changes to retained triggers. +`--triggers` skips these automatic pauses; it does not undo an operator's persisted pause. +SIGINT or SIGTERM shuts down the scheduler, API streams and location interpreter together. + ## Warehouse event identity Fingerprint and source-freshness writers persist deterministic `event_id` values in nullable text diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 038ceefb5b..dfd5c016a0 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,23 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "dev", + "parser": { + "prog": "sqb dev", + "description": "Run the local control API, scheduler and reloadable code location. Authentication is disabled only on loopback; triggers start paused. No UI is served.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": ["--project-dir", "--sqb-project-dir"], "dest": "project_dir", "default": {"suppress": true}, "help": "SQLBuild project directory", "group": 1}, + {"kind": "store", "options": ["--target"], "dest": "target", "help": "target to compile and run (default: configured target)", "group": 1}, + {"kind": "store", "options": ["--port"], "dest": "control_port", "default": 8000, "type": "int", "help": "loopback TCP port, or 0 to choose a free port (default: 8000)", "group": 1}, + {"kind": "store", "options": ["--control-store-url"], "dest": "development_store_url", "help": "control store URL or path (default: .sqlbuild/control.sqlite3 in the project)", "group": 1}, + {"kind": "store_true", "options": ["--no-scheduler"], "dest": "control_no_scheduler", "nargs": 0, "const": true, "default": false, "help": "serve and reload without executing scheduled work", "group": 1}, + {"kind": "store_true", "options": ["--triggers"], "dest": "development_triggers", "nargs": 0, "const": true, "default": false, "help": "preserve trigger defaults and operator overrides instead of pausing on startup", "group": 1} + ] + } + }, { "name": "location", "parser": { @@ -8582,6 +8599,11 @@ } ], "choices_help": [ + { + "dest": "dev", + "metavar": "dev", + "help": "run the loopback development API, scheduler and reloadable code location" + }, { "dest": "location", "metavar": "location", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 27c20ee723..6d982826ca 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -8,12 +8,16 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::api::models::DevelopmentScope; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink; use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; @@ -53,9 +57,140 @@ use crate::native_commands::models::{ LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; use crate::native_commands::types::LocationPublication; +use crate::workers::main::open_development_workers::open_development_workers; use crate::workers::main::open_workers::open_workers; +use crate::workers::main::terminate_child::terminate_child; use crate::workers::models::Workers; +/// Own the API, scheduler and outbound location as one development process tree. +pub(crate) async fn run_development_roles( + (store, scope): (ControlStore, DevelopmentScope), + (project, executable): (PathBuf, PathBuf), + listener: tokio::net::TcpListener, + scheduler: bool, +) -> Result<(), ControlError> { + let address = listener.local_addr()?; + let url = format!("http://{address}"); + let location_id = scope.location_id; + let target = scope.target.clone(); + let triggers = scope.triggers; + let workers = open_development_workers( + (location_id, target.clone()), + project.clone(), + executable.clone(), + url.clone(), + )?; + let (stop, shutdown) = watch::channel(false); + let (_changes, changes) = watch::channel(()); + let mut tasks = JoinSet::new(); + tasks.spawn(serve_api( + store.clone(), + listener, + Some(scope), + development_stopped(shutdown.clone()), + )); + start_platform_roles(&mut tasks, (&store, &shutdown))?; + tasks.spawn(development_location_role( + (&project, &executable), + (location_id, &target), + (&url, &store), + shutdown.clone(), + )?); + if scheduler { + drop(write_output(&[( + Stream::Stderr, + "Starting development scheduler...\n", + )])); + let scheduler_store = store.clone(); + tasks.spawn(async move { + run_scheduler( + &scheduler_store, + SchedulerState::new(Uuid::now_v7()), + (shutdown, changes), + workers, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Development scheduler started\n", + )])); + } + drop(write_output(&[( + Stream::Stderr, + &format!( + "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", + if triggers { + "follow code defaults and operator overrides" + } else { + "start paused" + } + ), + )])); + supervise_control_roles(tasks, stop).await +} + +async fn development_stopped(mut shutdown: watch::Receiver) { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } +} + +/// Hand off only the local file exclusion, never control-store connection credentials. +fn development_location_role( + (project, executable): (&Path, &Path), + (location_id, target): (Uuid, &str), + (url, store): (&str, &ControlStore), + shutdown: watch::Receiver, +) -> Result> + Send + 'static, ControlError> { + let mut command = Command::new(executable); + command + .args(["--project-dir"]) + .arg(project) + .args([ + "location", + "serve", + "--server", + url, + "--location-id", + &location_id.to_string(), + "--target", + target, + ]) + .env("SQLBUILD_LOCATION_TOKEN", "local-development") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .env_remove("SQLBUILD_PLATFORM_WEBHOOK_URL") + .env_remove("SQLBUILD_RUN_TOKEN") + .env_remove("SQLBUILD_DEVELOPMENT_CONTROL_FILE") + .stdin(Stdio::null()) + .kill_on_drop(true); + if let ControlStore::Sqlite(path) = store { + command.env( + "SQLBUILD_DEVELOPMENT_CONTROL_FILE", + std::env::current_dir()?.join(path), + ); + } + let mut child = command.spawn()?; + Ok(async move { + tokio::select! { + result = child.wait() => { + result?; + Err(ControlError::Invalid("development code location stopped unexpectedly".into())) + }, + () = development_stopped(shutdown) => { + terminate_child(&mut child)?; + match tokio::time::timeout(Duration::from_secs(10), child.wait()).await { + Ok(result) => { result?; }, + Err(_) => child.kill().await?, + } + Ok(()) + }, + } + }) +} + /// Request one publication without transferring interpreter ownership to a cancellable future. pub(crate) async fn compile_generation( requests: &mpsc::Sender, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 78b96ec94b..e64efaba99 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -19,10 +19,15 @@ use crate::replay::main::output_file_identities::output_file_identities; /// Observe the same project inputs as compiler reuse, excluding redirected command output. pub(crate) fn location_watch(root: &Path) -> LocationWatch { let rules = location_rules(); + let files: Vec = std::env::var_os("SQLBUILD_DEVELOPMENT_CONTROL_FILE") + .into_iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + let local_files = location_database_files(root, files); LocationWatch { - observed: location_inputs(root, &rules, &[]), + observed: location_inputs(root, &rules, &local_files), rules, - local_files: Vec::new(), + local_files, attempted: None, changed_at: tokio::time::Instant::now(), retry_at: None, diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs index e94539aa29..7b06b015ef 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -2,9 +2,14 @@ use std::path::Path; +use sqlbuild_config::effective::main::effective_target_name::effective_target_name; +use sqlbuild_config::effective::models::TargetChoice; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::project::main::load_local_config::load_local_config; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::models::{Built, ConfigPaths}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_orchestration::store::errors::ControlError; use crate::native_commands::constants::{ CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME, @@ -64,3 +69,41 @@ pub(crate) fn has_no_runtime_extensions(project_dir: &Path) -> bool { pub(crate) fn has_custom_rules(project_dir: &Path) -> bool { project_dir.join(CUSTOM_RULES_DIRECTORY).is_dir() } + +/// Resolve development's target through the same selected/local/default precedence as compilation. +pub(crate) fn development_target( + project_dir: &Path, + selected: Option, +) -> Result { + let paths = config_paths(project_dir); + let invalid = + || ControlError::Invalid("project configuration is invalid; run sqb debug".into()); + let project = load_project_config(&paths).map_err(|_| invalid())?; + let local = load_local_config(&paths).map_err(|_| invalid())?; + let text = |config: &Built, name: &str| match object_field(config, name) { + Some(Built::Value(ConfigValue::String(value))) => Some(value.clone()), + _ => None, + }; + let defined: Vec = [&project, &local] + .into_iter() + .filter_map(|config| object_field(config, "targets")) + .filter_map(|targets| match targets { + Built::Dict(targets) => Some(targets), + _ => None, + }) + .flatten() + .map(|(name, _)| name.clone()) + .collect(); + effective_target_name(TargetChoice { + selected, + local: text(&local, "target"), + default: text(&project, "default_target"), + defined, + }) + .map_err(|_| { + ControlError::Invalid("unknown development target; run sqb debug --target ".into()) + })? + .ok_or_else(|| { + ControlError::Invalid("sqb dev requires --target or a configured default target".into()) + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 02e6928bd2..cc58bab029 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -45,6 +45,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, + None, stopped(shutdown.clone()), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs new file mode 100644 index 0000000000..bbffbb5261 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs @@ -0,0 +1,82 @@ +//! Collapse the production API, scheduler and outbound location into one local process tree. + +use std::path::Path; + +use sqlbuild_orchestration::api::models::DevelopmentScope; +use sqlbuild_orchestration::credentials::main::development_location::development_location; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::declarations::main::pause_development_triggers::pause_development_triggers; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::{migrate_store::migrate_store, open_store::open_store}; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{control_address, flag, text}; +use crate::native_commands::_helpers::execution::run_development_roles; +use crate::native_commands::_helpers::project_facts::development_target; +use crate::native_commands::_helpers::terminal::write_output; + +pub(super) async fn run_dev(namespace: &Namespace) -> Result<(), ControlError> { + let project = + std::fs::canonicalize(text(namespace, "project_dir").unwrap_or_else(|| ".".into()))?; + let target = development_target(&project, text(namespace, "target"))?; + let address = control_address(namespace, false)? + .ok_or_else(|| ControlError::Invalid("development address is required".into()))?; + if !address.ip().is_loopback() { + return Err(ControlError::Invalid( + "sqb dev only listens on loopback".into(), + )); + } + let executable = std::env::current_exe()?; + drop(write_output(&[( + Stream::Stderr, + "Starting development control store...\n", + )])); + let (store, scope) = development_store(namespace, &project, &target).await?; + drop(write_output(&[( + Stream::Stderr, + "Development control store ready\nStarting development API...\n", + )])); + let listener = tokio::net::TcpListener::bind(address).await?; + run_development_roles( + (store, scope), + (project, executable), + listener, + !flag(namespace, "control_no_scheduler"), + ) + .await +} + +/// Complete retained identity and startup pauses before starting any scheduling roles. +async fn development_store( + namespace: &Namespace, + project: &Path, + target: &str, +) -> Result<(ControlStore, DevelopmentScope), ControlError> { + let database = text(namespace, "development_store_url").unwrap_or_else(|| { + project + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned() + }); + let store = open_store(&database).await?; + migrate_store(&store).await?; + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = + development_location(&store, &project.to_string_lossy(), target, &context).await?; + let scope = DevelopmentScope { + location_id, + target: target.into(), + triggers: flag(namespace, "development_triggers"), + }; + if !scope.triggers { + pause_development_triggers(&store, (location_id, target)).await?; + } + Ok((store, scope)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index af7c252502..04602f8d37 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -5,6 +5,7 @@ pub(super) mod _location_work; pub(super) mod _run_admin; pub(super) mod _run_control; pub(super) mod _run_cost; +pub(super) mod _run_dev; pub(super) mod _run_location; pub(crate) mod parse_native_invocation; pub(crate) mod run_control_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs index b9869af6a9..ac052a6b55 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs @@ -6,15 +6,24 @@ use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::text; use crate::native_commands::_helpers::execution::location_options; use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_run_dev::run_dev; use crate::native_commands::main::_run_location::run_location; pub(crate) fn run_location_invocation(namespace: &Namespace, script: &Path) -> Option { - if text(namespace, "command").as_deref() != Some("location") - || text(namespace, "location_command").as_deref() != Some("serve") + let development = text(namespace, "command").as_deref() == Some("dev"); + if !development + && (text(namespace, "command").as_deref() != Some("location") + || text(namespace, "location_command").as_deref() != Some("serve")) { return None; } let result = (|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + if development { + return runtime.block_on(run_dev(namespace)); + } let options = location_options(namespace)?; let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { std::io::Error::other( @@ -24,23 +33,31 @@ pub(crate) fn run_location_invocation(namespace: &Namespace, script: &Path) -> O if bearer.trim().is_empty() { return Err(std::io::Error::other("SQLBUILD_LOCATION_TOKEN must not be blank").into()); } - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; runtime.block_on(run_location(options, script, &bearer)) })(); Some(match result { Ok(()) => { drop(write_output(&[( Stream::Stderr, - "Code location stopped successfully\n", + if development { + "Development services stopped successfully\n" + } else { + "Code location stopped successfully\n" + }, )])); 0 } Err(error) => { drop(write_output(&[( Stream::Stderr, - &format!("Code location failed: {error}\n"), + &format!( + "{} failed: {error}\n", + if development { + "Development services" + } else { + "Code location" + } + ), )])); 1 } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs index 75d44e336f..c8c2643d4b 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/child.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -2,7 +2,8 @@ use sqlbuild_orchestration::store::errors::ControlError; -use crate::workers::_helpers::supervisor::{exit_code, terminate}; +use crate::workers::_helpers::supervisor::exit_code; +use crate::workers::main::terminate_child::terminate_child; use crate::workers::models::{WorkerChild, WorkerExit}; impl WorkerExit { @@ -42,7 +43,7 @@ impl WorkerChild { pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { match self { - Self::Local(child) => terminate(child), + Self::Local(child) => terminate_child(child), #[cfg(target_os = "linux")] Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM), Self::Docker(child) => child.signal("SIGTERM").await, diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index ab053c0d10..6cbfc0b2d2 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -19,7 +19,7 @@ use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; use sqlbuild_orchestration::scheduler::models::SchedulerLease; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::models::ControlStore; -use tokio::process::{Child, Command}; +use tokio::process::Command; use uuid::Uuid; use crate::workers::_helpers::invocation::{arguments, environment, excluded}; @@ -390,20 +390,3 @@ pub(in crate::workers) fn exit_code(exit: ExitStatus) -> i64 { #[cfg(not(unix))] i64::from(exit.code().unwrap_or(1)) } - -pub(in crate::workers) fn terminate(child: &mut Child) -> Result<(), ControlError> { - #[cfg(unix)] - if let Some(id) = child.id() { - let raw = i32::try_from(id) - .map_err(|error| ControlError::Invalid(format!("invalid child process ID: {error}")))?; - let pid = rustix::process::Pid::from_raw(raw) - .ok_or_else(|| ControlError::Invalid("invalid child process ID".into()))?; - match rustix::process::kill_process(pid, rustix::process::Signal::TERM) { - Ok(()) | Err(rustix::io::Errno::SRCH) => {} - Err(error) => return Err(std::io::Error::from(error).into()), - } - } - #[cfg(not(unix))] - let _ = child; - Ok(()) -} diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs index 2c35ad2bd4..def6c0f550 100644 --- a/crates/sqlbuild-cli/src/workers/main/mod.rs +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -1 +1,3 @@ +pub(crate) mod open_development_workers; pub(crate) mod open_workers; +pub(crate) mod terminate_child; diff --git a/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs new file mode 100644 index 0000000000..846eadcc68 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs @@ -0,0 +1,33 @@ +//! Development uses the ordinary local launcher without writing derived launcher configuration. + +use std::path::PathBuf; + +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::workers::models::{Launchers, LocalLauncher, Workers}; + +pub(crate) fn open_development_workers( + (location_id, target): (Uuid, String), + project_dir: PathBuf, + executable: PathBuf, + server_url: String, +) -> Result { + Ok(Workers { + configuration: Launchers { + local: vec![LocalLauncher { + location_id, + target, + server_url, + max_workers: std::thread::available_parallelism()?, + project_dir, + executable, + }], + docker: Vec::new(), + concurrency_limits: Default::default(), + }, + active: Vec::new(), + control_variable: "SQLBUILD_CONTROL_DATABASE_URL".into(), + recovered_epoch: None, + }) +} diff --git a/crates/sqlbuild-cli/src/workers/main/terminate_child.rs b/crates/sqlbuild-cli/src/workers/main/terminate_child.rs new file mode 100644 index 0000000000..1c2c371306 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/terminate_child.rs @@ -0,0 +1,21 @@ +//! Signal only a child whose process handle is still owned by the caller. + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::process::Child; + +pub(crate) fn terminate_child(child: &mut Child) -> Result<(), ControlError> { + #[cfg(unix)] + if let Some(id) = child.id() { + let raw = i32::try_from(id) + .map_err(|error| ControlError::Invalid(format!("invalid child process ID: {error}")))?; + let pid = rustix::process::Pid::from_raw(raw) + .ok_or_else(|| ControlError::Invalid("invalid child process ID".into()))?; + match rustix::process::kill_process(pid, rustix::process::Signal::TERM) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + #[cfg(not(unix))] + let _ = child; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 24f97c2b34..542bbd196b 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -13,6 +13,7 @@ chrono.workspace = true chrono-tz.workspace = true croner.workspace = true flate2.workspace = true +futures-util.workspace = true getrandom.workspace = true reqwest.workspace = true rusqlite.workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs index 70c607a995..1bee960538 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -1,13 +1,15 @@ //! Authenticate before body buffering; scope checks remain in each owning endpoint. +use std::net::SocketAddr; + use axum::extract::{Request, State}; -use axum::http::{HeaderValue, StatusCode, header}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; use axum::middleware::Next; use axum::response::Response; use uuid::Uuid; use crate::api::errors::ApiError; -use crate::api::models::ReadResource; +use crate::api::models::{DevelopmentScope, ReadResource}; use crate::credentials::main::authenticate_token::authenticate_token; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::store::main::{begin::begin, commit::commit, query::query}; @@ -18,20 +20,22 @@ pub(crate) async fn authenticate( mut request: Request, next: Next, ) -> Result { - let unauthorized = || ApiError { - status: StatusCode::UNAUTHORIZED, - message: "valid bearer credential required".into(), + let development = request.extensions().get::().cloned(); + let identity = match bearer(request.headers()) { + Ok(token) => authenticate_token(&store, token).await?, + Err(_) if development.is_some() => None, + Err(error) => return Err(error), }; - let bearer = request - .headers() - .get(header::AUTHORIZATION) - .ok_or_else(unauthorized)? - .to_str() - .map_err(|_| unauthorized())? - .strip_prefix("Bearer ") - .ok_or_else(unauthorized)?; - let identity = authenticate_token(&store, bearer) - .await? + let identity = identity + .or_else(|| { + development.map(|scope| TokenIdentity { + token_id: Uuid::nil(), + scope: CredentialScope::Location { + location_id: scope.location_id, + }, + expires_at: chrono::DateTime::::MAX_UTC, + }) + }) .ok_or_else(unauthorized)?; request.extensions_mut().insert(identity); let mut response = next.run(request).await; @@ -41,6 +45,51 @@ pub(crate) async fn authenticate( Ok(response) } +pub(crate) fn bearer(headers: &HeaderMap) -> Result<&str, ApiError> { + headers + .get(header::AUTHORIZATION) + .ok_or_else(unauthorized)? + .to_str() + .map_err(|_| unauthorized())? + .strip_prefix("Bearer ") + .ok_or_else(unauthorized) +} + +fn unauthorized() -> ApiError { + ApiError { + status: StatusCode::UNAUTHORIZED, + message: "valid bearer credential required".into(), + } +} + +/// Reject browser cross-origin and DNS-rebinding requests on the disabled-auth listener. +pub(crate) async fn local_request( + State(address): State, + request: Request, + next: Next, +) -> Result { + let authorities = [address.to_string(), format!("localhost:{}", address.port())]; + let valid_host = request.headers().get(header::HOST).is_some_and(|value| { + value + .to_str() + .is_ok_and(|host| authorities.iter().any(|allowed| host == allowed)) + }); + let valid_origin = request.headers().get(header::ORIGIN).is_none_or(|value| { + value.to_str().is_ok_and(|origin| { + authorities + .iter() + .any(|host| origin == format!("http://{host}")) + }) + }); + if !valid_host || !valid_origin { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "development API requires a local authority and origin".into(), + }); + } + Ok(next.run(request).await) +} + pub(crate) fn require_location( identity: &TokenIdentity, location_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/changes.rs new file mode 100644 index 0000000000..98a327eedd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/changes.rs @@ -0,0 +1,178 @@ +//! Scoped invalidations, not a replayable event log or a second source of application data. + +use std::time::Duration; + +use axum::Extension; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, HeaderValue}; +use axum::response::Sse; +use axum::response::sse::Event; +use futures_util::{Stream, stream}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use tokio::sync::broadcast; +use uuid::Uuid; + +use crate::api::_helpers::authentication::{bearer, require_location}; +use crate::api::errors::ApiError; +use crate::api::models::{ChangeFeed, ChangeNotice, ChangeStream}; +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::listen_changes::listen_changes; +use crate::store::models::{ChangeMessage, ChangeTopic, ControlStore}; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/changes", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = ChangeNotice, content_type = "text/event-stream", + description = "changed events: all, runs, run:, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor."), + (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn subscribe( + State(store): State, + (Extension(identity), Extension(feed)): (Extension, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + headers: HeaderMap, +) -> Result>>, ApiError> { + require_location(&identity, location_id)?; + let authorization = if identity.token_id.is_nil() { + None + } else { + let mut authorization = HeaderValue::from_str(bearer(&headers)?) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + authorization.set_sensitive(true); + Some(authorization) + }; + let state = ChangeStream { + store, + authorization, + location_id, + target_hash: sha256_hex(target.as_bytes()), + target, + receiver: feed.sender.subscribe(), + stopped: feed.stopped, + keepalive: tokio::time::interval(Duration::from_secs(15)), + initial: true, + }; + Ok(Sse::new(stream::try_unfold(state, next_event))) +} + +/// Revalidate every emission, keeping unexpected storage failures as stream errors. +async fn next_event( + mut state: ChangeStream, +) -> Result, ControlError> { + loop { + if *state.stopped.borrow() { + return Ok(None); + } + let message = if std::mem::take(&mut state.initial) { + Some(ChangeMessage::Resync) + } else { + tokio::select! { + _ = state.stopped.changed() => return Ok(None), + _ = state.keepalive.tick() => None, + message = state.receiver.recv() => match message { + Ok(message) => Some(message), + Err(broadcast::error::RecvError::Lagged(_)) => Some(ChangeMessage::Resync), + Err(broadcast::error::RecvError::Closed) => return Ok(None), + }, + } + }; + let keepalive = message.is_none(); + let topic = match message { + Some(ChangeMessage::Changed { + location_id, + target_hash, + topic, + }) => { + if location_id != state.location_id || target_hash != state.target_hash { + continue; + } + Some(match topic { + ChangeTopic::Runs => "runs".into(), + ChangeTopic::Run(id) => format!("run:{id}"), + ChangeTopic::Ticks => "ticks".into(), + ChangeTopic::Snapshot => format!("snapshot:{}", state.location_id), + ChangeTopic::NodeStatus => format!("node_status:{}", state.target), + }) + } + Some(ChangeMessage::Resync) => Some("all".into()), + Some(ChangeMessage::CredentialsChanged) | None => None, + }; + if let Some(authorization) = &state.authorization { + let token = authorization + .to_str() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let Some(identity) = authenticate_token(&state.store, token).await? else { + return Ok(None); + }; + if identity.scope + != (CredentialScope::Location { + location_id: state.location_id, + }) + { + return Ok(None); + } + } + if let Some(topic) = topic { + let event = Event::default() + .event("changed") + .json_data(ChangeNotice { topic }) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + return Ok(Some((event, state))); + } + if keepalive { + return Ok(Some((Event::default().comment("keep-alive"), state))); + } + } +} + +pub(crate) async fn relay( + store: ControlStore, + mut listener: Option, + sender: broadcast::Sender, +) -> Result<(), ControlError> { + let ControlStore::Postgres(pool) = store else { + return std::future::pending().await; + }; + loop { + if listener.is_none() { + tokio::time::sleep(Duration::from_secs(1)).await; + listener = match listen_changes(&pool).await { + Ok(listener) => { + let _ = sender.send(ChangeMessage::Resync); + Some(listener) + } + Err(ControlError::Database(error)) if transient_listener_error(&error) => continue, + Err(error) => return Err(error), + }; + } + let Some(active) = listener.as_mut() else { + continue; + }; + let message = match active.try_recv().await { + Ok(Some(notification)) => serde_json::from_str(notification.payload()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + Ok(None) => ChangeMessage::Resync, + Err(error) if transient_listener_error(&error) => { + listener = None; + continue; + } + Err(error) => return Err(error.into()), + }; + let _ = sender.send(message); + } +} + +fn transient_listener_error(error: &sqlx_core::Error) -> bool { + match error { + sqlx_core::Error::Io(_) + | sqlx_core::Error::PoolTimedOut + | sqlx_core::Error::WorkerCrashed => true, + sqlx_core::Error::Database(error) => error.code().is_some_and(|code| { + code.starts_with("08") + || code.starts_with("53") + || matches!(code.as_ref(), "57P01" | "57P02" | "57P03") + }), + _ => false, + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs index 60a407233c..65baf9a1a0 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs @@ -13,7 +13,7 @@ use crate::platform::main::report_location_health::report_location_health; use crate::platform::models::{LocationHealthObservation, LocationHealthReport}; use crate::store::models::ControlStore; -#[utoipa::path(post, path = "/api/locations/{location_id}/health", +#[utoipa::path(post, path = "/api/locations/{location_id}/health", operation_id = "report_location_health", params(("location_id" = Uuid, Path)), request_body = LocationHealthReport, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), @@ -30,7 +30,7 @@ pub(crate) async fn report( })) } -#[utoipa::path(get, path = "/api/locations/{location_id}/health", +#[utoipa::path(get, path = "/api/locations/{location_id}/health", operation_id = "read_location_health", params(("location_id" = Uuid, Path)), responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 94872aa28d..0f6623e274 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod authentication; +pub(crate) mod changes; pub(crate) mod client; pub(crate) mod location_health; pub(crate) mod location_work; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs index 06ee0574ab..44c484838a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs @@ -20,7 +20,7 @@ use crate::store::main::read_recall_view::read_recall_view; use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::store::types::Row; -#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", operation_id = "list_runs", params(("location_id" = Uuid, Path), ("target" = String, Path), ("limit" = Option, Query, description = "Maximum recent runs; defaults to 100, capped at 1000")), responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), @@ -72,7 +72,7 @@ pub(crate) async fn invocations( )) } -#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", operation_id = "read_run_recall", params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), ("view" = String, Path), ("invocation" = String, Query, description = "Exact scheduler or worker invocation ID"), ("limit" = Option, Query, description = "Maximum rows; defaults to 100, capped at 1000")), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs index 3762a428bf..636ccc0f9f 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -38,7 +38,7 @@ pub(crate) async fn execution( Ok(Json(execution)) } -#[utoipa::path(post, path = "/api/runs/{run_id}/events", params(("run_id" = Uuid, Path)), +#[utoipa::path(post, path = "/api/runs/{run_id}/events", operation_id = "report_run_event", params(("run_id" = Uuid, Path)), request_body = RunChangeRequest, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs index 0b6f56f2af..7c7e452aee 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -7,9 +7,11 @@ use uuid::Uuid; use crate::api::_helpers::authentication::require_location; use crate::api::errors::ApiError; +use crate::api::models::DevelopmentScope; use crate::credentials::models::TokenIdentity; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::current_snapshot_record::current_snapshot_record; +use crate::snapshots::main::publish_development_snapshot::publish_development_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; use crate::store::models::ControlStore; @@ -23,7 +25,10 @@ use crate::store::models::ControlStore; pub(crate) async fn publish( State(store): State, Extension(identity): Extension, - Path((location_id, target)): Path<(Uuid, String)>, + (Path((location_id, target)), development): ( + Path<(Uuid, String)>, + Option>, + ), Json(input): Json, ) -> Result, ApiError> { require_location(&identity, location_id)?; @@ -33,7 +38,22 @@ pub(crate) async fn publish( message: "snapshot does not match the requested location and target".into(), }); } - Ok(Json(publish_snapshot(&store, &input).await?)) + let pause_new = if let Some(Extension(scope)) = development { + if scope.location_id != location_id || scope.target != target { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "snapshot is outside the development scope".into(), + }); + } + !scope.triggers + } else { + false + }; + Ok(Json(if pause_new { + publish_development_snapshot(&store, &input).await? + } else { + publish_snapshot(&store, &input).await? + })) } #[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots", diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs index bcc5fd5bdd..dcced622c8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs @@ -15,7 +15,7 @@ use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::models::TickRecord; use crate::store::models::ControlStore; -#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", operation_id = "list_triggers", params(("location_id" = Uuid, Path), ("target" = String, Path)), responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), security(("bearer" = [])))] diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs index 5cca1d650f..4645faec67 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs @@ -13,7 +13,7 @@ use crate::warehouse_metadata::main::ingest_fingerprints::ingest_fingerprints; use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; use crate::warehouse_metadata::models::{FingerprintBatch, SourceFreshnessBatch}; -#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", operation_id = "report_source_freshness", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index cd9bc03bb5..319dad03bd 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -2,25 +2,41 @@ use axum::extract::DefaultBodyLimit; use axum::routing::{get, post}; -use axum::{Json, Router, middleware}; +use axum::{Extension, Json, Router, middleware}; -use crate::api::_helpers::authentication::authenticate; +use crate::api::_helpers::authentication::{authenticate, local_request}; use crate::api::_helpers::{ - location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, + changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, + warehouse_metadata, }; use crate::api::constants::BODY_BYTES; use crate::api::main::openapi_document::openapi_document; +use crate::api::models::{ChangeFeed, DevelopmentScope}; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; +use crate::store::main::subscribe_changes::subscribe_changes; use crate::store::models::ControlStore; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, + development_scope: Option, shutdown: impl Future + Send + 'static, ) -> Result<(), ControlError> { + let address = listener.local_addr()?; + validate_development_listener(address, &development_scope)?; migrate_store(&store).await?; + let (sender, listener_changes) = subscribe_changes(&store).await?; + let (stop, stopped) = tokio::sync::watch::channel(false); + let feed = ChangeFeed { + sender: sender.clone(), + stopped, + }; let protected = Router::new() + .route( + "/api/locations/{location_id}/targets/{target}/changes", + get(changes::subscribe), + ) .route( "/api/locations/{location_id}/work/poll", post(location_work::poll), @@ -83,15 +99,46 @@ pub async fn serve_api( .route("/api/runs/{run_id}/output", post(runs::output)) .route("/api/runs/{run_id}/recall", post(runs::recall)) .layer(DefaultBodyLimit::max(BODY_BYTES)) + .layer(Extension(feed)) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); let application = protected .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), ) - .with_state(store); - axum::serve(listener, application) - .with_graceful_shutdown(shutdown) - .await?; + .with_state(store.clone()); + let application = development_application(application, development_scope, address); + let server = axum::serve(listener, application).with_graceful_shutdown(async move { + shutdown.await; + stop.send_replace(true); + }); + tokio::select! { + result = async { server.await } => { result?; Ok(()) }, + result = changes::relay(store, listener_changes, sender) => result, + } +} + +fn validate_development_listener( + address: std::net::SocketAddr, + scope: &Option, +) -> Result<(), ControlError> { + if scope.is_some() && !address.ip().is_loopback() { + return Err(ControlError::Invalid( + "disabled authentication requires a loopback listener".into(), + )); + } Ok(()) } + +fn development_application( + application: Router, + scope: Option, + address: std::net::SocketAddr, +) -> Router { + match scope { + Some(scope) => application + .layer(Extension(scope)) + .layer(middleware::from_fn_with_state(address, local_request)), + None => application, + } +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 750c46b118..3b53aceaf0 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -3,11 +3,13 @@ use serde::{Deserialize, Serialize}; use crate::api::_helpers::{ - location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, + changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, + warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( + changes::subscribe, location_health::report, location_health::recent, location_work::poll, @@ -75,6 +77,40 @@ pub struct ApiErrorBody { pub error: String, } +/// SSE invalidation topics; `all` means refetch the authorized REST resources. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ChangeNotice { + pub topic: String, +} + +/// Explicit local development scope; never enabled by ordinary server startup. +#[derive(Clone, Debug)] +pub struct DevelopmentScope { + pub location_id: uuid::Uuid, + pub target: String, + pub triggers: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct ChangeFeed { + pub(crate) sender: tokio::sync::broadcast::Sender, + pub(crate) stopped: tokio::sync::watch::Receiver, +} + +/// Sensitive headers redact the bearer from Debug output and all wire notifications. +#[derive(Debug)] +pub(crate) struct ChangeStream { + pub(crate) store: crate::store::models::ControlStore, + pub(crate) authorization: Option, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) target_hash: String, + pub(crate) receiver: tokio::sync::broadcast::Receiver, + pub(crate) stopped: tokio::sync::watch::Receiver, + pub(crate) keepalive: tokio::time::Interval, + pub(crate) initial: bool, +} + #[derive(Clone, Debug)] pub(crate) struct ControlClient { pub(crate) client: reqwest::Client, diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs new file mode 100644 index 0000000000..672c0c5f70 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs @@ -0,0 +1,37 @@ +//! Create a server-owned location identity independent of its mutable label. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn create_location_in( + mut transaction: ControlTransaction, + name: &str, + context: &AuditContext, +) -> Result<(ControlTransaction, Uuid), ControlError> { + if name.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + let id = Uuid::now_v7(); + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; + let transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "location.created"), + ( + json!({"location_id": id}), + json!(null), + json!({"name": name}), + ), + ) + .await?; + Ok((transaction, id)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index ae89a5d233..5eea412ac1 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod locations; pub(crate) mod scopes; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs index a046c16405..39a04ea9f4 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs @@ -1,44 +1,21 @@ -//! Create a server-owned location identity independent of its mutable label. +//! Register a location and its security audit atomically. -use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::_helpers::locations::create_location_in; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; -/// Register a new location; names are labels, never credentials or stable identifiers. +/// Names are labels, never credentials or stable identifiers. pub async fn create_location( store: &ControlStore, name: &str, context: &AuditContext, ) -> Result { - if name.trim().is_empty() { - return Err(ControlError::Invalid("location name is required".into())); - } - let mut transaction = begin(store, TransactionMode::Write).await?; - let id = Uuid::now_v7(); - let now = transaction.now; - (transaction, _) = query(transaction, - "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", - vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; - commit( - append_audit( - transaction, - context, - (Uuid::new_v5(&id, b"created"), "location.created"), - ( - json!({"location_id": id}), - json!(null), - json!({"name": name}), - ), - ) - .await?, - ) - .await?; + let (transaction, id) = + create_location_in(begin(store, TransactionMode::Write).await?, name, context).await?; + commit(transaction).await?; Ok(id) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/development_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/development_location.rs new file mode 100644 index 0000000000..4049e823a3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/development_location.rs @@ -0,0 +1,53 @@ +//! A dev project/target retains its opaque location identity across process restarts. + +use uuid::Uuid; + +use crate::credentials::_helpers::locations::create_location_in; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn development_location( + store: &ControlStore, + project_path: &str, + target: &str, + context: &AuditContext, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Write).await?, + "SELECT location_id FROM sqb_development_locations WHERE project_path=$1 AND target=$2", + vec![ + Parameter::Text(project_path.into()), + Parameter::Text(target.into()), + ], + ) + .await?; + if let Some(row) = rows.first() { + let id = row["location_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid development location identity".into()))?; + commit(transaction).await?; + return Ok(id); + } + let (transaction, id) = create_location_in( + transaction, + &format!("development-{}", Uuid::now_v7()), + context, + ) + .await?; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_development_locations (project_path,target,location_id) VALUES ($1,$2,$3)", + vec![ + Parameter::Text(project_path.into()), + Parameter::Text(target.into()), + Parameter::Id(id), + ], + ) + .await?; + commit(transaction).await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index bde23d947a..ad09e37418 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -2,5 +2,6 @@ pub(crate) mod append_audit; pub mod authenticate_token; pub mod create_location; pub mod create_principal; +pub mod development_location; pub mod issue_token; pub mod revoke_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs index a7366783ae..76d619d5be 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -9,7 +9,7 @@ use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; /// Repeating a revocation does not create a second audit entry or rewrite its actor. pub async fn revoke_token( @@ -30,19 +30,18 @@ pub async fn revoke_token( .get("scope_json") .cloned() .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; - commit( - append_audit( - transaction, - context, - (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), - ( - scope, - json!({"token_id": token_id, "revoked": false}), - json!({"token_id": token_id, "revoked": true}), - ), - ) - .await?, + let mut transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), + ( + scope, + json!({"token_id": token_id, "revoked": false}), + json!({"token_id": token_id, "revoked": true}), + ), ) .await?; + transaction.changes.insert(Change::Credentials); + commit(transaction).await?; Ok(true) } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs new file mode 100644 index 0000000000..636f2afab1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs @@ -0,0 +1,63 @@ +//! Development safety defaults are ordinary audited trigger changes, not edited definitions. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn trigger_ids( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT trigger_id FROM sqb_triggers WHERE location_id=$1 AND target=$2 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ).await?; + let ids = rows + .iter() + .map(|row| { + row["trigger_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid trigger identity".into())) + }) + .collect::>()?; + Ok((transaction, ids)) +} + +pub(crate) async fn pause_triggers( + transaction: ControlTransaction, + scope: (Uuid, &str), + previous: &BTreeSet, + request_id: Uuid, +) -> Result { + let (mut transaction, ids) = trigger_ids(transaction, scope).await?; + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id, + ip_address: None, + }; + for trigger_id in ids.difference(previous) { + (transaction, _) = change_trigger_in( + transaction, + &context, + &TriggerChangeRequest { + event_id: Uuid::new_v5(&request_id, trigger_id.as_bytes()), + trigger_id: *trigger_id, + actor: context.actor.clone(), + reason: "Development starts with triggers paused".into(), + change: TriggerChange::Pause, + }, + ) + .await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index 75c383f6bc..1e3bd2943f 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1,3 +1,6 @@ +pub(crate) mod development; pub(crate) mod documents; +pub(crate) mod operator_change; pub(crate) mod operator_state; +pub(crate) mod publication; pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs new file mode 100644 index 0000000000..82ea7acec7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs @@ -0,0 +1,69 @@ +//! Apply an operator fact and its projection in one transaction, with retry-safe request identity. + +use serde_json::json; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::declarations::_helpers::operator_state::apply_operator_change; +use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn change_trigger_in( + transaction: ControlTransaction, + context: &AuditContext, + request: &TriggerChangeRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.actor.trim().is_empty() || request.actor != context.actor { + return Err(ControlError::Invalid( + "trigger changes require the authenticated audit actor".into(), + )); + } + let body = + serde_json::to_value(request).map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_trigger_state_events WHERE event_id=$1", + vec![Parameter::Id(request.event_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&body) { + return Err(ControlError::Invalid( + "trigger change event ID already has different content".into(), + )); + } + return Ok((transaction, false)); + } + let (transaction, rows) = query( + transaction, + "SELECT t.definition_json, t.snapshot_id, s.started_at, s.last_tick_for, s.status_override \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.trigger_id=$1 AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)], + ) + .await?; + let state: EditableTriggerState = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + let snapshot_id = state.snapshot_id; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", + vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; + let (transaction, (before, after)) = apply_operator_change(transaction, state, request).await?; + let transaction = append_audit( + transaction, + context, + (request.event_id, "trigger.changed"), + ( + json!({"trigger_id": request.trigger_id, "snapshot_id": snapshot_id}), + before, + after, + ), + ) + .await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs index b07ce5e73b..0bae32b69c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs @@ -10,7 +10,7 @@ use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; use crate::sensors::main::reset_sensor_cursor::reset_sensor_cursor; use crate::store::errors::ControlError; use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::{Change, ControlTransaction, Parameter}; pub(crate) async fn apply_operator_change( transaction: ControlTransaction, @@ -73,9 +73,12 @@ pub(crate) async fn apply_operator_change( }); } let now = transaction.now; - let (transaction, _) = query(transaction, + let (mut transaction, _) = query(transaction, "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + transaction + .changes + .insert(Change::Trigger(request.trigger_id)); Ok((transaction, (before, after))) } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs new file mode 100644 index 0000000000..69f65255bc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs @@ -0,0 +1,105 @@ +//! Publish definition projections inside the current-snapshot transaction. + +use uuid::Uuid; + +use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; +use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::sensors::main::publish_sensors::publish_sensors; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_projections( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: Definitions, +) -> Result { + transaction = publish_jobs(transaction, snapshot, &definitions).await?; + (transaction, _) = query(transaction, + "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + transaction = publish_sensors(transaction, snapshot, &definitions).await?; + for (name, mut trigger) in definitions.triggers { + let id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let previous; + (transaction, previous) = query(transaction, + "SELECT t.definition_json, s.started_at, s.last_tick_for, s.status_override FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id WHERE s.trigger_id=$1", + vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + retain_calendar_ticks(&name, &mut trigger, state.as_ref(), snapshot)?; + let definition = serde_json::to_value(&trigger) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ($1,$2,$3,'schedule',$4,$5,$6,$7) \ + ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", + vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), + Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; + let started_at = state.as_ref().map_or( + trigger.starts_at.unwrap_or(snapshot.published_at), + |state| state.started_at, + ); + let (status, next) = trigger_timing( + &trigger, + started_at, + state.as_ref().and_then(|state| state.last_tick_for), + state.as_ref().and_then(|state| state.status_override), + )?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", + vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} + +async fn publish_jobs( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + } + let jobs = definitions + .jobs + .iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .iter() + .map(|(name, group)| (name, "group", &group.job)), + ); + for (name, kind, job) in jobs { + let definition = + serde_json::to_value(job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; + for node in &job.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node.clone())]).await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs index 095353123a..365b5ce2d5 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -1,77 +1,24 @@ -//! Apply an operator fact and its projection in one transaction, with retry-safe request identity. +//! Commit an operator fact with its projection and security audit. -use serde_json::json; - -use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::declarations::_helpers::operator_state::apply_operator_change; -use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::models::TriggerChangeRequest; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::decode_row::decode_row; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; -/// Authorization belongs to the API boundary; `actor` is the authenticated principal, not client input. +/// Authorization belongs to the API boundary; `actor` is authenticated, not client input. pub async fn change_trigger( store: &ControlStore, context: &AuditContext, request: &TriggerChangeRequest, ) -> Result { - if request.actor.trim().is_empty() || request.actor != context.actor { - return Err(ControlError::Invalid( - "trigger changes require the authenticated audit actor".into(), - )); - } - let body = - serde_json::to_value(request).map_err(|error| ControlError::Invalid(error.to_string()))?; - let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = query( - transaction, - "SELECT request_json FROM sqb_trigger_state_events WHERE event_id=$1", - vec![Parameter::Id(request.event_id)], - ) - .await?; - if let Some(previous) = previous.first() { - if previous.get("request_json") != Some(&body) { - return Err(ControlError::Invalid( - "trigger change event ID already has different content".into(), - )); - } - commit(transaction).await?; - return Ok(false); - } - let (transaction, rows) = query( - transaction, - "SELECT t.definition_json, t.snapshot_id, s.started_at, s.last_tick_for, s.status_override \ - FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ - WHERE t.trigger_id=$1 AND t.removed_in_snapshot_id IS NULL", - vec![Parameter::Id(request.trigger_id)], - ) - .await?; - let state: EditableTriggerState = decode_row(rows.into_iter().next().ok_or_else(|| { - ControlError::Invalid("trigger does not exist in the current snapshot".into()) - })?)?; - let snapshot_id = state.snapshot_id; - let now = transaction.now; - let (transaction, _) = query(transaction, - "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", - vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; - let (transaction, (before, after)) = apply_operator_change(transaction, state, request).await?; - commit( - append_audit( - transaction, - context, - (request.event_id, "trigger.changed"), - ( - json!({"trigger_id": request.trigger_id, "snapshot_id": snapshot_id}), - before, - after, - ), - ) - .await?, + let (transaction, applied) = change_trigger_in( + begin(store, TransactionMode::Write).await?, + context, + request, ) .await?; - Ok(true) + commit(transaction).await?; + Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index d87701b504..4e0773fdd4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -3,5 +3,6 @@ pub(crate) mod frozen_definitions; pub mod group_dependencies; pub mod list_trigger_events; pub mod list_triggers; +pub mod pause_development_triggers; pub(crate) mod publish_definitions; pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs new file mode 100644 index 0000000000..5c8c182541 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs @@ -0,0 +1,18 @@ +//! Pause retained triggers before a development scheduler can acquire leadership. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::_helpers::development::pause_triggers; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn pause_development_triggers( + store: &ControlStore, + scope: (Uuid, &str), +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + commit(pause_triggers(transaction, scope, &BTreeSet::new(), Uuid::now_v7()).await?).await +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index 230f3c116d..e6dfefd53b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -1,105 +1,35 @@ -//! Publish definition projections inside the current-snapshot transaction. +//! Publish definition projections and optional audited development defaults atomically. -use uuid::Uuid; +use std::collections::BTreeSet; -use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; -use crate::declarations::models::{Definitions, PreviousTriggerState}; -use crate::sensors::main::publish_sensors::publish_sensors; +use crate::declarations::_helpers::development::{pause_triggers, trigger_ids}; +use crate::declarations::_helpers::publication::publish_projections; +use crate::declarations::models::Definitions; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; -use crate::store::main::decode_row::decode_row; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::ControlTransaction; pub(crate) async fn publish_definitions( - mut transaction: ControlTransaction, + transaction: ControlTransaction, snapshot: &SnapshotRecord, definitions: Definitions, + pause_new_triggers: bool, ) -> Result { - transaction = publish_jobs(transaction, snapshot, &definitions).await?; - (transaction, _) = query(transaction, - "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", - vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; - transaction = publish_sensors(transaction, snapshot, &definitions).await?; - for (name, mut trigger) in definitions.triggers { - let id = Uuid::new_v5( - &snapshot.location_id, - &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) - .map_err(|error| ControlError::Invalid(error.to_string()))?, - ); - let previous; - (transaction, previous) = query(transaction, - "SELECT t.definition_json, s.started_at, s.last_tick_for, s.status_override FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id WHERE s.trigger_id=$1", - vec![Parameter::Id(id)]).await?; - let state = previous - .into_iter() - .next() - .map(decode_row::) - .transpose()?; - retain_calendar_ticks(&name, &mut trigger, state.as_ref(), snapshot)?; - let definition = serde_json::to_value(&trigger) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ($1,$2,$3,'schedule',$4,$5,$6,$7) \ - ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", - vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), - Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; - let started_at = state.as_ref().map_or( - trigger.starts_at.unwrap_or(snapshot.published_at), - |state| state.started_at, - ); - let (status, next) = trigger_timing( - &trigger, - started_at, - state.as_ref().and_then(|state| state.last_tick_for), - state.as_ref().and_then(|state| state.status_override), - )?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ - ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", - vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; - } - Ok(transaction) -} - -async fn publish_jobs( - mut transaction: ControlTransaction, - snapshot: &SnapshotRecord, - definitions: &Definitions, -) -> Result { - for table in ["sqb_job_nodes", "sqb_jobs"] { - (transaction, _) = query( + let (transaction, previous) = if pause_new_triggers { + trigger_ids(transaction, (snapshot.location_id, &snapshot.target)).await? + } else { + (transaction, BTreeSet::new()) + }; + let transaction = publish_projections(transaction, snapshot, definitions).await?; + if pause_new_triggers { + pause_triggers( transaction, - &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), - vec![ - Parameter::Id(snapshot.location_id), - Parameter::Text(snapshot.target.clone()), - ], + (snapshot.location_id, &snapshot.target), + &previous, + snapshot.snapshot_id, ) - .await?; - } - let jobs = definitions - .jobs - .iter() - .map(|(name, job)| (name, "job", job)) - .chain( - definitions - .groups - .iter() - .map(|(name, group)| (name, "group", &group.job)), - ); - for (name, kind, job) in jobs { - let definition = - serde_json::to_value(job).map_err(|error| ControlError::Invalid(error.to_string()))?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", - vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), - Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; - for node in &job.selection { - (transaction, _) = query(transaction, - "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", - vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node.clone())]).await?; - } + .await + } else { + Ok(transaction) } - Ok(transaction) } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs index a146a8e4a7..2bd4d871bf 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; -use crate::store::models::{ControlEvent, ControlTransaction}; +use crate::store::models::{Change, ControlEvent, ControlTransaction}; pub(crate) async fn append_run_event( transaction: ControlTransaction, @@ -22,7 +22,7 @@ pub(crate) async fn append_run_event( run_id: Some(run_id), payload, }; - let (transaction, outcome) = append_control_event(transaction, event).await?; + let (mut transaction, outcome) = append_control_event(transaction, event).await?; if outcome.conflict.is_some() { return Err(ControlError::Invalid( "run event conflicts with a durable fact".into(), @@ -31,5 +31,8 @@ pub(crate) async fn append_run_event( let cursor = outcome .storage_id .ok_or_else(|| ControlError::Invalid("run event has no storage cursor".into()))?; + if outcome.inserted { + transaction.changes.insert(Change::Run(run_id)); + } Ok((transaction, cursor)) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs index 17e4e0c7a3..8e7fce4680 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -9,7 +9,7 @@ use crate::store::errors::ControlError; use crate::store::main::append_recall::append_recall; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::{Change, ControlStore, TransactionMode}; /// The active-claim check and the complete batch commit share a serialized write transaction. pub async fn publish_run_output( @@ -31,6 +31,9 @@ pub async fn publish_run_output( published.duplicates += 1; } } + if published.inserted > 0 || published.conflicts > 0 { + transaction.changes.insert(Change::Run(run_id)); + } commit(transaction).await?; Ok(published) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs index 30c14bb9d8..967e89ad14 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -10,7 +10,7 @@ use crate::runs::models::RecallBatch; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::{Change, ControlStore, TransactionMode}; pub async fn publish_run_recall( store: &ControlStore, @@ -33,7 +33,10 @@ pub async fn publish_run_recall( .iter() .map(|record| claim_record(&envelope, record)) .collect::, _>>()?; - let (transaction, result) = append_records(transaction, &envelope, &rows).await?; + let (mut transaction, result) = append_records(transaction, &envelope, &rows).await?; + if !rows.is_empty() { + transaction.changes.insert(Change::Run(identity.0)); + } commit(transaction).await?; Ok(result) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 0f659487c0..cafcbb9a7c 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -8,7 +8,7 @@ use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; use crate::store::errors::ControlError; use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::{Change, ControlTransaction, Parameter}; pub(crate) async fn persist_ticks( transaction: ControlTransaction, @@ -35,6 +35,11 @@ pub(crate) async fn persist_ticks( VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", vec![Parameter::Id(action.tick_id), Parameter::Id(trigger.trigger_id), Parameter::Time(action.scheduled_for), Parameter::Time(now), Parameter::Integer(lease.epoch), Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered), Parameter::Json(json!({}))]).await?; + if !inserted.is_empty() { + transaction + .changes + .insert(Change::Trigger(trigger.trigger_id)); + } if let Some(run_id) = run && !inserted.is_empty() { diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs index c94e5577c5..c7408021a1 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -9,7 +9,7 @@ use crate::sensors::models::{ use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; use crate::store::main::query::query; -use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; +use crate::store::models::{Change, ControlEvent, ControlTransaction, Parameter}; pub(crate) async fn append_evaluation( transaction: ControlTransaction, @@ -45,11 +45,14 @@ pub(crate) async fn append_evaluation( let storage_id = appended .storage_id .ok_or_else(|| ControlError::Invalid("sensor evaluation has no storage cursor".into()))?; - let (transaction, _) = query(transaction, + let (mut transaction, _) = query(transaction, "INSERT INTO sqb_sensor_evaluation_events (event_id,storage_id,evaluation_id,trigger_id,phase,occurred_at,fact_json) \ VALUES ($1,$2,$3,$4,$5,$6,$7)", vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(request.evaluation_id), Parameter::Id(request.trigger_id), Parameter::Text(phase.into()), Parameter::Time(occurred_at), Parameter::Json(payload)]).await?; + transaction + .changes + .insert(Change::Trigger(request.trigger_id)); Ok((transaction, (event_id, storage_id))) } diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs new file mode 100644 index 0000000000..7829a8a40e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs @@ -0,0 +1,97 @@ +//! Publish a complete generation and its current pointer in one transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::declarations::main::publish_definitions::publish_definitions; +use crate::declarations::main::snapshot_definitions::snapshot_definitions; +use crate::snapshots::_helpers::preparation::prepare_snapshot; +use crate::snapshots::_helpers::publication::{store_changes, store_nodes}; +use crate::snapshots::models::{PreparedSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn publish_snapshot_with_defaults( + store: &ControlStore, + input: &SnapshotInput, + pause_new_triggers: bool, +) -> Result { + let definitions = snapshot_definitions(input)?; + let PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + } = prepare_snapshot(input)?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query(transaction, + "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + if previous + .first() + .and_then(|row| row.get("generation")) + .and_then(Value::as_i64) + .is_some_and(|generation| generation >= input.generation) + { + return Err(ControlError::StaleGeneration); + } + let (mut transaction, previous) = query(transaction, + "SELECT n.node_key, n.node_hash FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + let previous: BTreeMap = previous + .into_iter() + .map(|row| { + ( + row["node_key"].as_str().unwrap_or_default().into(), + row["node_hash"].as_str().unwrap_or_default().into(), + ) + }) + .collect(); + let record = SnapshotRecord { + snapshot_id: Uuid::now_v7(), + location_id: input.location_id, + target: input.target.clone(), + generation: input.generation, + manifest_digest, + published_at: transaction.now, + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_revision, git_dirty, git_author, image_id, \ + environment_hash, sqb_version, snapshot_schema_version, manifest_digest, published_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,1,$11,$12)", vec![ + Parameter::Id(record.snapshot_id), Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Integer(input.generation), Parameter::OptionalText(input.git_revision.clone()), Parameter::Boolean(input.git_dirty), + Parameter::OptionalText(input.git_author.clone()), Parameter::OptionalText(input.image_id.clone()), + Parameter::Text(input.environment_hash.clone()), Parameter::Text(input.sqb_version.clone()), + Parameter::Text(record.manifest_digest.clone()), Parameter::Time(record.published_at), + ]).await?; + transaction = store_nodes(transaction, record.snapshot_id, nodes).await?; + for (kind, (hash, body)) in documents { + (transaction, _) = query(transaction, + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; + } + transaction = store_changes(transaction, record.snapshot_id, &previous, &hashes).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; + transaction = + publish_definitions(transaction, &record, definitions, pause_new_triggers).await?; + transaction + .changes + .insert(Change::Snapshot(input.location_id, input.target.clone())); + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 6918426ec2..f89796c34a 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,6 +1,8 @@ +pub(super) mod _publish_snapshot; pub mod current_snapshot; pub mod current_snapshot_record; pub mod next_remote_generation; +pub(crate) mod publish_development_snapshot; pub mod publish_remote_snapshot; pub mod publish_snapshot; pub(crate) mod read_snapshot_documents; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs new file mode 100644 index 0000000000..9d3065c812 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs @@ -0,0 +1,13 @@ +//! Publication keeps development safety defaults inside the snapshot transaction. + +use crate::snapshots::main::_publish_snapshot::publish_snapshot_with_defaults; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn publish_development_snapshot( + store: &ControlStore, + input: &SnapshotInput, +) -> Result { + publish_snapshot_with_defaults(store, input, true).await +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs index c34cf82ab9..471a2bebe6 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -1,93 +1,14 @@ -//! Publish a complete generation and its current pointer in one transaction. +//! Publish a generation using ordinary production trigger defaults. -use std::collections::BTreeMap; - -use serde_json::Value; -use uuid::Uuid; - -use crate::declarations::main::publish_definitions::publish_definitions; -use crate::declarations::main::snapshot_definitions::snapshot_definitions; -use crate::snapshots::_helpers::preparation::prepare_snapshot; -use crate::snapshots::_helpers::publication::{store_changes, store_nodes}; -use crate::snapshots::models::{PreparedSnapshot, SnapshotInput, SnapshotRecord}; +use crate::snapshots::main::_publish_snapshot::publish_snapshot_with_defaults; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::ControlStore; /// Reject stale generations without changing the last good snapshot. pub async fn publish_snapshot( store: &ControlStore, input: &SnapshotInput, ) -> Result { - let definitions = snapshot_definitions(input)?; - let PreparedSnapshot { - nodes, - hashes, - documents, - manifest_digest, - } = prepare_snapshot(input)?; - let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, previous) = query(transaction, - "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ - WHERE c.location_id=$1 AND c.target=$2", - vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; - if previous - .first() - .and_then(|row| row.get("generation")) - .and_then(Value::as_i64) - .is_some_and(|generation| generation >= input.generation) - { - return Err(ControlError::StaleGeneration); - } - let (mut transaction, previous) = query(transaction, - "SELECT n.node_key, n.node_hash FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ - WHERE c.location_id=$1 AND c.target=$2", - vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; - let previous: BTreeMap = previous - .into_iter() - .map(|row| { - ( - row["node_key"].as_str().unwrap_or_default().into(), - row["node_hash"].as_str().unwrap_or_default().into(), - ) - }) - .collect(); - let record = SnapshotRecord { - snapshot_id: Uuid::now_v7(), - location_id: input.location_id, - target: input.target.clone(), - generation: input.generation, - manifest_digest, - published_at: transaction.now, - }; - (transaction, _) = query(transaction, - "INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_revision, git_dirty, git_author, image_id, \ - environment_hash, sqb_version, snapshot_schema_version, manifest_digest, published_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,1,$11,$12)", vec![ - Parameter::Id(record.snapshot_id), Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), - Parameter::Integer(input.generation), Parameter::OptionalText(input.git_revision.clone()), Parameter::Boolean(input.git_dirty), - Parameter::OptionalText(input.git_author.clone()), Parameter::OptionalText(input.image_id.clone()), - Parameter::Text(input.environment_hash.clone()), Parameter::Text(input.sqb_version.clone()), - Parameter::Text(record.manifest_digest.clone()), Parameter::Time(record.published_at), - ]).await?; - transaction = store_nodes(transaction, record.snapshot_id, nodes).await?; - for (kind, (hash, body)) in documents { - (transaction, _) = query(transaction, - "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", - vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", - vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; - } - transaction = store_changes(transaction, record.snapshot_id, &previous, &hashes).await?; - (transaction, _) = query(transaction, - "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ - ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", - vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), - Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; - transaction = publish_definitions(transaction, &record, definitions).await?; - commit(transaction).await?; - Ok(record) + publish_snapshot_with_defaults(store, input, false).await } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs new file mode 100644 index 0000000000..cd23dd0052 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs @@ -0,0 +1,87 @@ +//! Commit-scoped invalidations; authoritative records remain in their existing stores. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use tokio::sync::broadcast; + +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ + Change, ChangeMessage, ChangeTopic, ChangedScope, ControlTransaction, Parameter, +}; + +/// Weak entries do not keep deleted test databases or idle subscribers alive. +pub(crate) fn sqlite_changes( + path: &Path, +) -> Result, ControlError> { + static CHANNELS: OnceLock>>> = + OnceLock::new(); + let path = path.canonicalize()?; + let mut channels = CHANNELS + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + channels.retain(|_, sender| sender.strong_count() > 0); + if let Some(sender) = channels.get(&path).and_then(broadcast::WeakSender::upgrade) { + return Ok(sender); + } + let (sender, _) = broadcast::channel(256); + channels.insert(path, sender.downgrade()); + Ok(sender) +} + +pub(crate) async fn resolve_changes( + mut transaction: ControlTransaction, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let mut messages: BTreeSet = BTreeSet::new(); + for change in std::mem::take(&mut transaction.changes) { + if change == Change::Credentials { + messages.insert(ChangeMessage::CredentialsChanged); + continue; + } + let scope = match &change { + Change::Snapshot(location_id, target) | Change::NodeStatus(location_id, target) => { + ChangedScope { + location_id: *location_id, + target: target.clone(), + } + } + Change::Run(id) | Change::Trigger(id) => { + let sql = if matches!(change, Change::Run(_)) { + "SELECT location_id, target FROM sqb_runs WHERE run_id=$1" + } else { + "SELECT location_id, target FROM sqb_triggers WHERE trigger_id=$1" + }; + let rows; + (transaction, rows) = query(transaction, sql, vec![Parameter::Id(*id)]).await?; + let Some(row) = rows.into_iter().next() else { + continue; + }; + decode_row(row)? + } + Change::Credentials => continue, + }; + let topics = match change { + Change::Run(id) => vec![ + ChangeTopic::Runs, + ChangeTopic::Run(id), + ChangeTopic::NodeStatus, + ], + Change::Trigger(_) => vec![ChangeTopic::Ticks], + Change::Snapshot(..) => vec![ChangeTopic::Snapshot, ChangeTopic::NodeStatus], + Change::NodeStatus(..) => vec![ChangeTopic::NodeStatus], + Change::Credentials => continue, + }; + for topic in topics { + messages.insert(ChangeMessage::Changed { + location_id: scope.location_id, + target_hash: sha256_hex(scope.target.as_bytes()), + topic, + }); + } + } + Ok((transaction, messages)) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 02c12874a0..8e208c08f5 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -16,6 +16,7 @@ pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshness.sql"); pub(crate) const WAREHOUSE_FINGERPRINTS: &str = include_str!("../warehouse_fingerprints.sql"); +pub(crate) const DEVELOPMENT_LOCATIONS: &str = include_str!("../development_locations.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs index 08fcf1b81c..1c71cbb1a9 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod changes; pub(crate) mod database; pub(crate) mod migration; pub(crate) mod open_sqlite; diff --git a/crates/sqlbuild-orchestration/src/store/development_locations.sql b/crates/sqlbuild-orchestration/src/store/development_locations.sql new file mode 100644 index 0000000000..afb397ba39 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/development_locations.sql @@ -0,0 +1,6 @@ +CREATE TABLE sqb_development_locations ( + project_path TEXT NOT NULL, + target TEXT NOT NULL, + location_id {uuid} NOT NULL UNIQUE REFERENCES sqb_locations(location_id), + PRIMARY KEY (project_path, target) +); diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index e97c51e077..4c86b0da5a 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -5,6 +5,7 @@ use std::time::Duration; use chrono::Utc; use rusqlite::OpenFlags; +use crate::store::_helpers::changes::sqlite_changes; use crate::store::errors::ControlError; use crate::store::main::refresh_clock::refresh_clock; use crate::store::models::{Connection, ControlStore, ControlTransaction, TransactionMode}; @@ -13,27 +14,31 @@ pub(crate) async fn begin( store: &ControlStore, mode: TransactionMode, ) -> Result { - let connection = match store { + let (connection, notifications) = match store { ControlStore::Sqlite(path) => { let path = path.clone(); - Connection::Sqlite( - tokio::task::spawn_blocking(move || { - let connection = rusqlite::Connection::open_with_flags( - path, - OpenFlags::SQLITE_OPEN_READ_WRITE - | OpenFlags::SQLITE_OPEN_CREATE - | OpenFlags::SQLITE_OPEN_NO_MUTEX, - )?; - connection.busy_timeout(Duration::from_secs(30))?; - connection.execute_batch(if mode == TransactionMode::Write { - "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" - } else { - "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" - })?; - Ok::<_, ControlError>(connection) - }) - .await??, - ) + let (connection, notifications) = tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch(if mode == TransactionMode::Write { + "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } else { + "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + })?; + let notifications = if mode == TransactionMode::Write { + Some(sqlite_changes(&path)?) + } else { + None + }; + Ok::<_, ControlError>((connection, notifications)) + }) + .await??; + (Connection::Sqlite(connection), notifications) } ControlStore::Postgres(pool) => { let mut transaction = pool.begin().await?; @@ -48,12 +53,14 @@ pub(crate) async fn begin( .execute(&mut *transaction) .await?; } - Connection::Postgres(transaction) + (Connection::Postgres(transaction), None) } }; let transaction = ControlTransaction { connection: Some(connection), now: Utc::now(), + changes: Default::default(), + notifications, }; refresh_clock(transaction).await } diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs index f9e3cb46e0..cc72f730ed 100644 --- a/crates/sqlbuild-orchestration/src/store/main/commit.rs +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -1,18 +1,41 @@ //! Commit facts and their projections together. +use crate::store::_helpers::changes::resolve_changes; use crate::store::errors::ControlError; use crate::store::models::{Connection, ControlTransaction}; -pub(crate) async fn commit(mut transaction: ControlTransaction) -> Result<(), ControlError> { +pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), ControlError> { + let (mut transaction, messages) = resolve_changes(transaction).await?; match transaction .connection .take() .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))? { Connection::Sqlite(connection) => { - tokio::task::spawn_blocking(move || connection.execute_batch("COMMIT")).await??; + tokio::task::spawn_blocking(move || { + connection.execute_batch("COMMIT")?; + if let Some(sender) = transaction.notifications { + for message in messages { + let _ = sender.send(message); + } + } + Ok::<_, rusqlite::Error>(()) + }) + .await??; + } + Connection::Postgres(mut transaction) => { + for message in messages { + let payload = serde_json::to_string(&message) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + sqlx_core::query::query( + "SELECT pg_notify('sqb_changes_' || md5(current_schema()), $1)", + ) + .bind(payload) + .execute(&mut *transaction) + .await?; + } + transaction.commit().await?; } - Connection::Postgres(transaction) => transaction.commit().await?, } Ok(()) } diff --git a/crates/sqlbuild-orchestration/src/store/main/listen_changes.rs b/crates/sqlbuild-orchestration/src/store/main/listen_changes.rs new file mode 100644 index 0000000000..f4a2711a54 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/listen_changes.rs @@ -0,0 +1,15 @@ +//! Establish schema-isolated PostgreSQL LISTEN before subscribing clients or declaring recovery. + +use crate::store::errors::ControlError; + +pub(crate) async fn listen_changes( + pool: &sqlx_postgres::PgPool, +) -> Result { + let channel: String = + sqlx_core::query_scalar::query_scalar("SELECT 'sqb_changes_' || md5(current_schema())") + .fetch_one(pool) + .await?; + let mut listener = sqlx_postgres::PgListener::connect_with(pool).await?; + listener.listen(&channel).await?; + Ok(listener) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index c3b4a99f16..895f5e2304 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,8 +4,9 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, - SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, + DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, + render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -38,6 +39,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("node_sensors", NODE_SENSORS), ("warehouse_freshness", WAREHOUSE_FRESHNESS), ("warehouse_fingerprints", WAREHOUSE_FINGERPRINTS), + ("development_locations", DEVELOPMENT_LOCATIONS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index e0d83e510a..9796d266c4 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -6,9 +6,11 @@ pub(crate) mod compress; pub(crate) mod decode_row; pub(crate) mod decompress; pub mod list_recall_invocations; +pub(crate) mod listen_changes; pub mod migrate_store; pub mod open_store; pub(crate) mod query; pub mod read_recall_view; pub(crate) mod refresh_clock; pub(crate) mod store_source_text; +pub(crate) mod subscribe_changes; diff --git a/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs b/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs new file mode 100644 index 0000000000..1513d649ce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs @@ -0,0 +1,33 @@ +//! SQLite shares an in-process channel; PostgreSQL listens across independent server replicas. + +use tokio::sync::broadcast; + +use crate::store::_helpers::changes::sqlite_changes; +use crate::store::errors::ControlError; +use crate::store::main::listen_changes::listen_changes; +use crate::store::models::{ChangeMessage, ControlStore}; + +pub(crate) async fn subscribe_changes( + store: &ControlStore, +) -> Result< + ( + broadcast::Sender, + Option, + ), + ControlError, +> { + match store { + ControlStore::Sqlite(path) => { + let path = path.clone(); + Ok(( + tokio::task::spawn_blocking(move || sqlite_changes(&path)).await??, + None, + )) + } + ControlStore::Postgres(pool) => { + let listener = listen_changes(pool).await?; + let (sender, _) = broadcast::channel(256); + Ok((sender, Some(listener))) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index 414bca79cd..4f1ddeb631 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -1,10 +1,13 @@ //! Native database handles and bound values shared by control-plane operations. +use std::collections::BTreeSet; use std::path::PathBuf; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; use sqlbuild_recall::schema::models::ColumnType; use sqlbuild_recall::store::models::ConflictEvidence; +use tokio::sync::broadcast; use uuid::Uuid; /// An installation's control store. Workers and locations never receive this handle. @@ -45,6 +48,8 @@ pub(crate) enum TransactionMode { pub(crate) struct ControlTransaction { pub(crate) connection: Option, pub(crate) now: DateTime, + pub(crate) changes: BTreeSet, + pub(crate) notifications: Option>, } #[derive(Debug)] @@ -53,6 +58,43 @@ pub(crate) enum Connection { Postgres(sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>), } +/// Transaction-local invalidations, not another durable lifecycle or event log. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Change { + Run(Uuid), + Trigger(Uuid), + Snapshot(Uuid, String), + NodeStatus(Uuid, String), + Credentials, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ChangeTopic { + Runs, + Run(Uuid), + Ticks, + Snapshot, + NodeStatus, +} + +/// Fixed-size routing payloads fit PostgreSQL NOTIFY even with long target labels. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ChangeMessage { + Changed { + location_id: Uuid, + target_hash: String, + topic: ChangeTopic, + }, + CredentialsChanged, + Resync, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ChangedScope { + pub(crate) location_id: Uuid, + pub(crate) target: String, +} + /// Parameters keep their PostgreSQL types; SQLite uses their corresponding storage classes. #[derive(Clone, Debug)] pub(crate) enum Parameter { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs index a2a9a4e44c..e923464cf0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs @@ -8,7 +8,7 @@ use crate::store::errors::ControlError; use crate::store::main::{ commit::commit, compress::compress, query::query, store_source_text::store_source_text, }; -use crate::store::models::{ControlStore, Parameter}; +use crate::store::models::{Change, ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::fingerprints::{source_event_id, validate_batch}; use crate::warehouse_metadata::_helpers::ingestion::{begin_ingestion, scoped_event_id}; use crate::warehouse_metadata::models::FingerprintBatch; @@ -70,6 +70,11 @@ pub async fn ingest_fingerprints( Parameter::Json(record_json), Parameter::Time(received_at)]).await?; applied |= !inserted.is_empty(); } + if applied { + transaction + .changes + .insert(Change::NodeStatus(location_id, target.into())); + } commit(transaction).await?; Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index 9a7f93572b..cb39441bee 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -8,7 +8,7 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::commit::commit; use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter}; +use crate::store::models::{Change, ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::ingestion::{begin_ingestion, scoped_event_id}; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::models::SourceFreshnessBatch; @@ -89,6 +89,11 @@ pub async fn ingest_source_freshness( Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at)]).await?; applied |= !inserted.is_empty(); } + if applied { + transaction + .changes + .insert(Change::NodeStatus(location_id, target.into())); + } commit(transaction).await?; Ok(applied) } diff --git a/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs new file mode 100644 index 0000000000..9a7ff3e5d9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs @@ -0,0 +1,172 @@ +//! Real HTTP event parsing and commit-time fault injection shared by SSE contracts. + +use std::collections::BTreeSet; +use std::time::Duration; + +use uuid::Uuid; + +use crate::api::tests::test_types::{ResetProxy, SseConnection, TestApi}; +use crate::store::models::ControlStore; + +pub(crate) async fn connect( + client: &reqwest::Client, + api: &TestApi, + (location_id, target, bearer): (Uuid, &str, &str), +) -> SseConnection { + let response = client + .get(format!( + "{}/api/locations/{location_id}/targets/{target}/changes", + api.url + )) + .bearer_auth(bearer) + .send() + .await + .expect("open scoped SSE connection"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.headers()["content-type"], "text/event-stream"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let mut stream = SseConnection { + response, + pending: Vec::new(), + }; + assert_eq!( + next_topic(&mut stream).await.as_deref(), + Some("all"), + "a reconnect must refetch REST state" + ); + stream +} + +pub(crate) async fn next_topic(stream: &mut SseConnection) -> Option { + tokio::time::timeout(Duration::from_secs(3), async { + for _ in 0..4096 { + for _ in 0..4096 { + let Some(end) = stream.pending.windows(2).position(|bytes| bytes == b"\n\n") else { + break; + }; + let bytes: Vec<_> = stream.pending.drain(..end + 2).collect(); + let frame = std::str::from_utf8(&bytes).expect("UTF-8 event frame"); + let Some(data) = frame.lines().find_map(|line| line.strip_prefix("data: ")) else { + continue; + }; + assert!(frame.lines().any(|line| line == "event: changed")); + let body: serde_json::Value = + serde_json::from_str(data).expect("typed change notice"); + return Some(body["topic"].as_str().expect("topic").to_owned()); + } + let chunk = stream + .response + .chunk() + .await + .expect("SSE body remains readable")?; + stream.pending.extend_from_slice(&chunk); + } + panic!("SSE stream exhausted bounded frame reads") + }) + .await + .expect("bounded SSE delivery or shutdown") +} + +pub(crate) async fn expect_topics(stream: &mut SseConnection, expected: &[String]) { + let mut actual = BTreeSet::new(); + for _ in expected { + assert!( + actual.insert(next_topic(stream).await.expect("committed change")), + "transaction-local topics deduplicate" + ); + } + assert_eq!(actual, expected.iter().cloned().collect()); +} + +pub(crate) async fn reject_snapshot_commit(store: &ControlStore) { + execute_fault_sql(store, ( + "CREATE TABLE sse_commit_parent (id INTEGER PRIMARY KEY); \ + CREATE TABLE sse_commit_guard (id INTEGER REFERENCES sse_commit_parent(id) DEFERRABLE INITIALLY DEFERRED); \ + CREATE TRIGGER sse_reject_snapshot AFTER INSERT ON sqb_snapshots BEGIN INSERT INTO sse_commit_guard VALUES (1); END;", + "CREATE TABLE sse_commit_parent (id INTEGER PRIMARY KEY); \ + CREATE TABLE sse_commit_guard (id INTEGER REFERENCES sse_commit_parent(id) DEFERRABLE INITIALLY DEFERRED); \ + CREATE FUNCTION sse_reject_snapshot() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO sse_commit_guard VALUES (1); RETURN NEW; END $$; \ + CREATE TRIGGER sse_reject_snapshot AFTER INSERT ON sqb_snapshots FOR EACH ROW EXECUTE FUNCTION sse_reject_snapshot();" + )).await; +} + +pub(crate) async fn restore_snapshot_commits(store: &ControlStore) { + execute_fault_sql(store, ( + "DROP TRIGGER sse_reject_snapshot; DROP TABLE sse_commit_guard; DROP TABLE sse_commit_parent;", + "DROP TRIGGER sse_reject_snapshot ON sqb_snapshots; DROP FUNCTION sse_reject_snapshot(); DROP TABLE sse_commit_guard; DROP TABLE sse_commit_parent;" + )).await; +} + +async fn execute_fault_sql(store: &ControlStore, (sqlite, postgres): (&str, &str)) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(sqlite) + .expect("SQLite commit fault setup"); + return; + }; + sqlx_core::raw_sql::raw_sql(postgres) + .execute(pool) + .await + .expect("PostgreSQL commit fault setup"); +} + +/// TCP RST, unlike a PostgreSQL termination message, exercises SQLx's unhandled reset error. +pub(crate) async fn reset_proxy(store: &ControlStore) -> ResetProxy { + let ControlStore::Postgres(pool) = store else { + panic!("PostgreSQL fixture") + }; + let options = pool.connect_options().as_ref().clone(); + let upstream = (options.get_host().to_owned(), options.get_port()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("proxy socket"); + let port = listener.local_addr().expect("proxy address").port(); + let (reset, resets) = tokio::sync::watch::channel(0_u64); + let (shutdown, mut stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let mut connections = tokio::task::JoinSet::new(); + for _ in 0..10_000 { + tokio::select! { + accepted = listener.accept() => { + let (mut downstream, _) = accepted.expect("accept database connection"); + let upstream = upstream.clone(); + let mut reset = resets.clone(); + reset.borrow_and_update(); + connections.spawn(async move { + let mut upstream = tokio::net::TcpStream::connect(upstream).await.expect("connect real PostgreSQL"); + downstream.set_zero_linger().expect("reset rather than EOF on drop"); + tokio::select! { + _ = tokio::io::copy_bidirectional(&mut downstream, &mut upstream) => {}, + _ = reset.changed() => {}, + } + }); + }, + Some(result) = connections.join_next(), if !connections.is_empty() => { result.expect("proxy connection task"); }, + _ = &mut stopped => return, + } + } + panic!("database proxy exhausted bounded connection events"); + }); + let store = ControlStore::Postgres( + sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .connect_with( + options + .host("127.0.0.1") + .port(port) + .ssl_mode(sqlx_postgres::PgSslMode::Disable), + ) + .await + .expect("connect through reset proxy"), + ); + ResetProxy { + store, + reset, + shutdown, + task, + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 9743c15603..a927f62e60 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -1,17 +1,25 @@ use std::time::Duration; use crate::api::main::serve_api::serve_api; +use crate::api::models::DevelopmentScope; use crate::api::tests::test_types::TestApi; use crate::store::models::ControlStore; pub(crate) async fn start_api(store: &ControlStore) -> TestApi { + start_api_with_scope(store, None).await +} + +pub(crate) async fn start_api_with_scope( + store: &ControlStore, + scope: Option, +) -> TestApi { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("isolated API socket"); let url = format!("http://{}", listener.local_addr().expect("API address")); let (shutdown, signal) = tokio::sync::oneshot::channel(); let store = store.clone(); - let task = tokio::spawn(serve_api(store, listener, async { + let task = tokio::spawn(serve_api(store, listener, scope, async { let _ = signal.await; })); let client = reqwest::Client::builder() diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs new file mode 100644 index 0000000000..f679fc3b92 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -0,0 +1,419 @@ +//! Real SQLite broadcast and PostgreSQL LISTEN/NOTIFY feed independent HTTP observers. + +use std::time::Duration; + +use chrono::Utc; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::changes_helpers::{ + connect, expect_topics, next_topic, reject_snapshot_commit, reset_proxy, + restore_snapshot_commits, +}; +use crate::api::tests::helpers::{read_json, start_api, stop_api}; +use crate::api::tests::test_types::ChangesTestCase; +use crate::credentials::main::{issue_token::issue_token, revoke_token::revoke_token}; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::snapshot; +use crate::scheduler::models::TickDecision; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_independent_http_observers_when_committed_state_changes_then_scoped_sse_refetches_real_state_and_stops_on_revocation() + { + let test_cases = [ + ChangesTestCase { + description: "SQLite committed live updates", + backend: Backend::Sqlite, + expected_generation: 2, + }, + ChangesTestCase { + description: "PostgreSQL replicated live updates", + backend: Backend::Postgres, + expected_generation: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("isolated store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("location token"); + let mut input = snapshot(location_id); + input.documents.insert("jobs".into(), json!({"orders": {"description":"Build orders", "selection":["orders"], "overlap":"allow"}})); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description":"Refresh orders", "job_key":"orders", "starts_at":"2020-01-01T00:00:00Z", + "schedule":{"kind":"calendar", "times":["2020-01-01T00:00:00Z","2030-01-01T00:00:00Z"], "refresh_at":"2029-01-01T00:00:00Z"}}})); + publish_snapshot(store, &input) + .await + .expect("initial snapshot"); + let first = start_api(store).await; + let second = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("HTTP client"); + let scope = (location_id, "development", token.bearer.as_str()); + let mut a = connect(&client, &first, scope).await; + let mut b = connect(&client, &second, scope).await; + let path = format!( + "{}/api/locations/{location_id}/targets/development", + first.url + ); + assert_eq!( + client + .get(format!("{path}/changes")) + .send() + .await + .expect("anonymous request") + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(format!( + "{}/api/locations/{other}/targets/development/changes", + first.url + )) + .bearer_auth(&token.bearer) + .send() + .await + .expect("foreign subscription") + .status(), + reqwest::StatusCode::FORBIDDEN + ); + let mut foreign = input.clone(); + foreign.location_id = other; + publish_snapshot(store, &foreign) + .await + .expect("foreign publication"); + foreign.location_id = location_id; + foreign.target = "preview".into(); + publish_snapshot(store, &foreign) + .await + .expect("other target"); + input.generation = 2; + let response = client + .post(format!("{path}/snapshots")) + .bearer_auth(&token.bearer) + .json(&input) + .send() + .await + .expect("HTTP publish"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + let snapshot_topics = [ + format!("snapshot:{location_id}"), + "node_status:development".into(), + ]; + expect_topics(&mut a, &snapshot_topics).await; + expect_topics(&mut b, &snapshot_topics).await; + let current = read_json( + &client, + ( + &format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + second.url + ), + &token.bearer, + ), + ) + .await; + assert_eq!( + current["record"]["generation"], test_case.expected_generation, + "{}: notification observes committed state on the other replica", + test_case.description + ); + + reject_snapshot_commit(store).await; + input.generation = 3; + assert!( + publish_snapshot(store, &input).await.is_err(), + "late deferred constraint rejects COMMIT after all snapshot work" + ); + restore_snapshot_commits(store).await; + let current = read_json(&client, (&format!("{path}/snapshots"), &token.bearer)).await; + assert_eq!(current["record"]["generation"], 2); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "orders".into(), + reason: "manual verification".into(), + priority: 0, + window: None, + }; + assert!( + request_job_run(store, &audit, &request) + .await + .expect("queue job") + ); + let run_topics = [ + "runs".into(), + format!("run:{}", request.run_id), + "node_status:development".into(), + ]; + expect_topics(&mut a, &run_topics).await; + expect_topics(&mut b, &run_topics).await; + assert!( + !request_job_run(store, &audit, &request) + .await + .expect("idempotent retry") + ); + let trigger_id = list_triggers(store, location_id, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("scheduler elected"); + let plan = evaluate_trigger(store, &lease, trigger_id) + .await + .expect("tick decision") + .expect("active trigger"); + let TickDecision::Requested { run_id } = plan.actions[0].decision else { + panic!("new scheduled run") + }; + let tick_topics = [ + "ticks".into(), + "runs".into(), + format!("run:{run_id}"), + "node_status:development".into(), + ]; + expect_topics(&mut a, &tick_topics).await; + expect_topics(&mut b, &tick_topics).await; + change_trigger( + store, + &audit, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: audit.actor.clone(), + reason: "pause verification".into(), + change: TriggerChange::Pause, + }, + ) + .await + .expect("pause"); + expect_topics(&mut a, &["ticks".into()]).await; + expect_topics(&mut b, &["ticks".into()]).await; + + assert!( + revoke_token(store, &audit, token.identity.token_id) + .await + .expect("revoke") + ); + assert_eq!( + next_topic(&mut a).await, + None, + "revocation closes an existing stream" + ); + assert_eq!( + next_topic(&mut b).await, + None, + "revocation reaches another replica" + ); + let fresh = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("new token"); + let mut live = + connect(&client, &first, (location_id, "development", &fresh.bearer)).await; + let expiring = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::seconds(1), + ) + .await + .expect("expiring token"); + let mut expires = connect( + &client, + &second, + (location_id, "development", &expiring.bearer), + ) + .await; + tokio::time::sleep( + (expiring.identity.expires_at - Utc::now()) + .to_std() + .expect("token is still active") + + Duration::from_millis(10), + ) + .await; + publish_snapshot(store, &input) + .await + .expect("committed change after expiry"); + expect_topics(&mut live, &snapshot_topics).await; + assert_eq!( + next_topic(&mut expires).await, + None, + "expired streams disclose no subsequent changes" + ); + // Reconnect starts with all, then REST supplies the current state rather than a replay. + let mut reconnected = connect( + &client, + &second, + (location_id, "development", &fresh.bearer), + ) + .await; + let current = read_json(&client, (&format!("{path}/snapshots"), &fresh.bearer)).await; + assert_eq!(current["record"]["generation"], 3); + let long_target = "p".repeat(9_000); + let mut long_stream = + connect(&client, &second, (location_id, &long_target, &fresh.bearer)).await; + input.target = long_target.clone(); + publish_snapshot(store, &input) + .await + .expect("target larger than PostgreSQL's NOTIFY payload limit"); + expect_topics( + &mut long_stream, + &[ + format!("snapshot:{location_id}"), + format!("node_status:{long_target}"), + ], + ) + .await; + stop_api(first).await; + assert_eq!( + next_topic(&mut live).await, + None, + "shutdown drains a live SSE body" + ); + stop_api(second).await; + assert_eq!(next_topic(&mut reconnected).await, None); + assert_eq!(next_topic(&mut long_stream).await, None); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_terminated_postgres_listeners_when_reconnecting_then_http_survives_and_observers_refetch() +{ + let test_cases = [ChangesTestCase { + description: "PostgreSQL listener interruption", + backend: Backend::Postgres, + expected_generation: 2, + }]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("isolated store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("token"); + let mut input = snapshot(location_id); + publish_snapshot(store, &input).await.expect("snapshot"); + let proxy = reset_proxy(store).await; + let first = start_api(&proxy.store).await; + let second = start_api(&proxy.store).await; + let client = reqwest::Client::new(); + let scope = (location_id, "development", token.bearer.as_str()); + let mut a = connect(&client, &first, scope).await; + let mut b = connect(&client, &second, scope).await; + let ControlStore::Postgres(pool) = store else { + panic!("PostgreSQL fixture") + }; + let terminated: Vec = sqlx_core::query_scalar::query_scalar( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid<>pg_backend_pid() AND query LIKE 'LISTEN%' AND query LIKE '%' || md5(current_schema()) || '%'", + ).fetch_all(pool).await.expect("interrupt only this fixture's LISTEN connections"); + assert_eq!(terminated, [true, true]); + let path = format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + first.url + ); + let current = read_json(&client, (&path, &token.bearer)).await; + assert_eq!( + current["record"]["generation"], 1, + "HTTP still serves during listener recovery" + ); + expect_topics(&mut a, &["all".into()]).await; + expect_topics(&mut b, &["all".into()]).await; + proxy.reset.send_modify(|epoch| *epoch += 1); + expect_topics(&mut a, &["all".into()]).await; + expect_topics(&mut b, &["all".into()]).await; + let current = read_json(&client, (&path, &token.bearer)).await; + assert_eq!( + current["record"]["generation"], 1, + "HTTP survives a TCP reset" + ); + input.generation = 2; + publish_snapshot(store, &input) + .await + .expect("commit after re-listening"); + let topics = [ + format!("snapshot:{location_id}"), + "node_status:development".into(), + ]; + expect_topics(&mut a, &topics).await; + expect_topics(&mut b, &topics).await; + let mut reconnected = connect(&client, &second, scope).await; + let current = read_json(&client, (&path, &token.bearer)).await; + assert_eq!( + current["record"]["generation"], test_case.expected_generation, + "{}", + test_case.description + ); + stop_api(first).await; + stop_api(second).await; + assert_eq!(next_topic(&mut a).await, None); + assert_eq!(next_topic(&mut b).await, None); + assert_eq!(next_topic(&mut reconnected).await, None); + crate::store::tests::store_helpers::close_store(proxy.store).await; + proxy.shutdown.send(()).expect("stop database proxy"); + proxy.task.await.expect("database proxy stopped"); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index bffc493c5c..615e3cd663 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -1,13 +1,19 @@ //! HTTP contracts compiled within the API owner. +#[path = "changes/helpers.rs"] +mod changes_helpers; #[path = "helpers.rs"] pub(crate) mod helpers; #[path = "output/helpers.rs"] mod output_helpers; #[path = "recall/helpers.rs"] mod recall_helpers; +#[path = "test_changes.rs"] +mod test_changes; #[path = "test_contract.rs"] mod test_contract; +#[path = "test_development.rs"] +mod test_development; #[path = "test_location_health.rs"] mod test_location_health; #[path = "test_location_work.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_development.rs b/crates/sqlbuild-orchestration/tests/api/test_development.rs new file mode 100644 index 0000000000..11f5256375 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_development.rs @@ -0,0 +1,207 @@ +//! Development defaults share production transactions, identities and operator facts on both stores. + +use uuid::Uuid; + +use crate::api::models::DevelopmentScope; +use crate::credentials::main::development_location::development_location; +use crate::credentials::models::AuditContext; +use crate::declarations::main::{ + change_trigger::change_trigger, list_triggers::list_triggers, + pause_development_triggers::pause_development_triggers, +}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; +use crate::snapshots::main::{ + current_snapshot::current_snapshot, publish_snapshot::publish_snapshot, +}; +use crate::store::main::migrate_store::migrate_store; + +use crate::api::tests::helpers::{start_api_with_scope, stop_api}; +use crate::api::tests::test_types::DevelopmentTestCase; +use crate::credentials::tests::helpers::location_state; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_development_when_defaults_publish_then_identity_audit_and_trigger_pauses_commit_atomically() + { + let test_cases = [ + DevelopmentTestCase { + description: "SQLite atomic development defaults", + backend: Backend::Sqlite, + expected_status: TriggerStatus::Paused, + }, + DevelopmentTestCase { + description: "PostgreSQL atomic development defaults", + backend: Backend::Postgres, + expected_status: TriggerStatus::Paused, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let fault = TransactionFault { + description: "development audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected development audit failure')", + postgres_body: "RAISE EXCEPTION 'injected development audit failure'", + }; + install_fault(store, &fault).await; + assert!( + development_location(store, "/synthetic/orders", "development", &context) + .await + .is_err() + ); + assert_eq!(location_state(store).await, (0, 0)); + remove_fault(store, &fault).await; + let location = + development_location(store, "/synthetic/orders", "development", &context) + .await + .expect("create identity"); + assert_eq!( + development_location(store, "/synthetic/orders", "development", &context) + .await + .expect("retained identity"), + location + ); + let other = development_location(store, "/synthetic/orders", "preview", &context) + .await + .expect("target-specific identity"); + assert_ne!(location, other); + let api = start_api_with_scope( + store, + Some(DevelopmentScope { + location_id: location, + target: "development".into(), + triggers: false, + }), + ) + .await; + let client = reqwest::Client::new(); + let url = format!( + "{}/api/locations/{location}/targets/development/snapshots", + api.url + ); + let mut input = snapshot(location); + install_fault(store, &fault).await; + assert_eq!( + client + .post(&url) + .json(&input) + .send() + .await + .expect("rejected publication") + .status(), + reqwest::StatusCode::INTERNAL_SERVER_ERROR + ); + assert!( + current_snapshot(store, location, "development") + .await + .expect("last good snapshot") + .is_none() + ); + assert!( + list_triggers(store, location, "development") + .await + .expect("no leaked trigger") + .is_empty() + ); + remove_fault(store, &fault).await; + client + .post(&url) + .json(&input) + .send() + .await + .expect("publication") + .error_for_status() + .expect("publication succeeds after rollback"); + let triggers = list_triggers(store, location, "development") + .await + .expect("paused projection"); + assert_eq!(triggers.len(), 1); + assert_eq!( + triggers[0].status, test_case.expected_status, + "{}", + test_case.description + ); + change_trigger( + store, + &context, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id: triggers[0].trigger_id, + actor: context.actor.clone(), + reason: "Explicit development enable".into(), + change: TriggerChange::Enable, + }, + ) + .await + .expect("operator can enable retained triggers"); + input.generation = 2; + let additional = input.documents["triggers"]["daily_orders"].clone(); + input + .documents + .get_mut("triggers") + .expect("trigger document")["additional_orders"] = additional; + client + .post(&url) + .json(&input) + .send() + .await + .expect("reload") + .error_for_status() + .expect("atomic default pause on reload"); + let triggers = list_triggers(store, location, "development") + .await + .expect("reloaded triggers"); + assert_eq!( + triggers + .iter() + .map(|trigger| (trigger.name.as_str(), trigger.status)) + .collect::>(), + vec![ + ("additional_orders", TriggerStatus::Paused), + ("daily_orders", TriggerStatus::Running) + ] + ); + let mut other_input = snapshot(other); + other_input.target = "preview".into(); + publish_snapshot(store, &other_input) + .await + .expect("ordinary defaults stay enabled"); + pause_development_triggers(store, (location, "development")) + .await + .expect("restart pauses retained triggers"); + assert!( + list_triggers(store, location, "development") + .await + .expect("restart state") + .iter() + .all(|trigger| trigger.status == TriggerStatus::Paused) + ); + assert_eq!( + list_triggers(store, other, "preview") + .await + .expect("other scope unchanged")[0] + .status, + TriggerStatus::Running + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 0e84bbc2c1..133fd8774c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,10 +46,12 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_development_locations").await; execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_development_locations").await; execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; execute(store, "DROP TABLE sqb_warehouse_freshness").await; execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 71c81b400d..088a8ac099 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -8,6 +8,12 @@ pub(crate) struct ApiContractTestCase { pub(crate) expected_status: RunStatus, } +pub(crate) struct DevelopmentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: crate::declarations::models::TriggerStatus, +} + pub(crate) struct WorkerRedirectTestCase { pub(crate) description: &'static str, pub(crate) expected_status: reqwest::StatusCode, @@ -33,6 +39,24 @@ pub(crate) struct TestApi { pub(crate) task: tokio::task::JoinHandle>, } +pub(crate) struct SseConnection { + pub(crate) response: reqwest::Response, + pub(crate) pending: Vec, +} + +pub(crate) struct ResetProxy { + pub(crate) store: crate::store::models::ControlStore, + pub(crate) reset: tokio::sync::watch::Sender, + pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, + pub(crate) task: tokio::task::JoinHandle<()>, +} + +pub(crate) struct ChangesTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_generation: i64, +} + pub(crate) struct LocationHealthTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 5869626d42..1b2c5aa8a1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -5,6 +5,7 @@ use reqwest::StatusCode; use serde_json::json; use uuid::Uuid; +use crate::api::tests::changes_helpers::{connect, expect_topics}; use crate::api::tests::helpers::{start_api, stop_api}; use crate::api::tests::test_types::WarehouseMetadataTestCase; use crate::credentials::main::issue_token::issue_token; @@ -118,15 +119,22 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit let outbound = connect_location_work((&api.url, &token.bearer), location) .expect("pooled native location client"); let typed: SourceFreshnessBatch = serde_json::from_value(batch.clone()).expect("page"); - for expected_applied in [true, false] { - assert_eq!( - outbound - .sync_source_freshness("development", &typed) - .await - .expect("native client sync"), - expected_applied - ); - } + let mut changes = + connect(&client, &api, (location, "development", &token.bearer)).await; + assert!( + outbound + .sync_source_freshness("development", &typed) + .await + .expect("native client sync") + ); + expect_topics(&mut changes, &["node_status:development".into()]).await; + assert!( + !outbound + .sync_source_freshness("development", &typed) + .await + .expect("native client replay") + ); + drop(changes); let mut equivalent = batch.clone(); equivalent["records"] = json!([record]); equivalent["records"][0]["observed_at"] = json!("2026-01-01T11:00:00+01:00"); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 0e0ddc0487..f8951d639c 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -23,14 +23,17 @@ pub(crate) async fn fingerprint_history( // Upgrade the immediate predecessor while published snapshots and freshness facts exist. let (transaction, _) = query( begin(store, TransactionMode::Write).await.expect("upgrade"), - "DROP TABLE sqb_warehouse_fingerprints", + "DROP TABLE sqb_development_locations", vec![], ) .await - .expect("predecessor table set"); + .expect("remove subsequent development migration"); + let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_fingerprints", vec![]) + .await + .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name='warehouse_fingerprints'", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index 9e6edc1c94..edb65f4446 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -1,7 +1,7 @@ //! Credential contracts compiled within the credential owner. #[path = "helpers.rs"] -mod helpers; +pub(crate) mod helpers; #[path = "locations/helpers.rs"] pub(crate) mod location_helpers; #[path = "test_contract.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs index 57d98cb7cf..c70bd73153 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -7,4 +7,4 @@ mod test_contract; #[path = "test_types.rs"] mod test_types; #[path = "ticks/test_ticks.rs"] -mod ticks; +pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index 3f36d1d4fe..1b9c0ad341 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -1,5 +1,5 @@ #[path = "helpers.rs"] -mod helpers; +pub(crate) mod helpers; #[path = "test_contract.rs"] mod test_contract; #[path = "test_loop.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 1d3b9b0cf5..67f59043d0 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,10 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP TABLE sqb_development_locations", vec![]) + .await + .expect("remove subsequent development migration"); let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_fingerprints", vec![]) .await diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 86546af384..91f6eb9314 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ + "DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs index 988e9542be..7f0f9098dd 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -5,6 +5,19 @@ use crate::store::tests::store_helpers::run; use crate::store::tests::store_test_types::Backend; use crate::store::tests::test_types::RecallTestCase; +use std::time::Duration; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, + store_source_text::store_source_text, subscribe_changes::subscribe_changes, +}; +use crate::store::models::{Change, ChangeMessage, ChangeTopic, Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store}; +use crate::store::tests::test_types::CommitCancellationTestCase; + #[test] fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { let test_cases = [ @@ -28,3 +41,83 @@ fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_orde ); } } + +#[test] +fn given_sqlite_commit_cancellation_when_blocking_work_finishes_then_persisted_changes_still_notify() + { + let test_cases = [CommitCancellationTestCase { + description: "cancelled async caller cannot strand committed invalidations", + expected_body: b"SELECT 7 AS order_id", + }]; + for test_case in test_cases { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .expect("single blocking worker runtime") + .block_on(async { + let fixture = open_test_store(Backend::Sqlite) + .await + .expect("isolated SQLite"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let (sender, _) = subscribe_changes(store).await.expect("change subscription"); + let mut receiver = sender.subscribe(); + let hash = sha256_hex(test_case.expected_body); + let mut transaction = store_source_text( + begin(store, TransactionMode::Write) + .await + .expect("write transaction"), + (hash.clone(), test_case.expected_body.to_vec()), + ) + .await + .expect("insert source"); + transaction + .changes + .insert(Change::NodeStatus(Uuid::now_v7(), "development".into())); + + let (started, started_signal) = tokio::sync::oneshot::channel(); + let (release, wait) = std::sync::mpsc::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).expect("blocking worker occupied"); + let _ = wait.recv(); + }); + started_signal.await.expect("worker ready"); + let mut committing = Box::pin(commit(transaction)); + assert!( + std::future::poll_fn(|context| std::task::Poll::Ready( + committing.as_mut().poll(context).is_pending() + )) + .await + ); + drop(committing); + release.send(()).expect("release queued commit"); + blocker.await.expect("blocking worker released"); + let notice = tokio::time::timeout(Duration::from_secs(3), receiver.recv()) + .await + .expect("committed write still notifies") + .expect("live notification channel"); + assert!(matches!( + notice, + ChangeMessage::Changed { + topic: ChangeTopic::NodeStatus, + .. + } + )); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("read committed source"), + "SELECT body FROM sqb_source_texts WHERE text_hash=$1", + vec![Parameter::Text(hash)], + ) + .await + .expect("query committed bytes"); + commit(transaction).await.expect("finish read"); + let body: Vec = + serde_json::from_value(rows[0]["body"].clone()).expect("stored bytes"); + assert_eq!(body, test_case.expected_body, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs index 53cbf105a6..490a7f2c0c 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -13,3 +13,8 @@ pub(crate) struct RecallViewTestCase { pub(crate) backend: Backend, pub(crate) expected_complete: i64, } + +pub(crate) struct CommitCancellationTestCase { + pub(crate) description: &'static str, + pub(crate) expected_body: &'static [u8], +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 20b540641f..efb06e04fa 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -166,3 +166,13 @@ class FingerprintHistoryTestCase: journal_modes: tuple[str, ...] fixture_paths: tuple[Path, ...] expected_legacy_runs: tuple[str, ...] + + +@dataclass(frozen=True) +class DevelopmentTestCase: + description: str + target: str + arguments: tuple[str, ...] + control_path: str + expected_rows: list[tuple[int, int]] + local_config: str = "" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index ae1526b8c8..a1386b584f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -67,6 +67,14 @@ def location_process( _ = process.wait(timeout=5) +def stop_location(process: subprocess.Popen[bytes], interpreter_pid: int, log: Path) -> None: + """Exercise graceful supervisor shutdown and prove its interpreter was reaped.""" + process.send_signal(signal.SIGTERM) + assert process.wait(timeout=15) == 0, log.read_text() + with pytest.raises(ProcessLookupError): + os.kill(interpreter_pid, 0) + + def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) -> tuple[Any, Any]: """The caller owns the returned real connection; declarations must never run during reads.""" write_files( @@ -526,10 +534,11 @@ def wait_for_run( pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), job_key: str | None = None, run_id: str | None = None, + target: str = "development", ) -> dict[str, Any]: """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" request: Request = Request( - f"{api.url}/api/locations/{api.location_id}/targets/development/runs", + f"{api.url}/api/locations/{api.location_id}/targets/{quote(target, safe='')}/runs", headers={"Authorization": f"Bearer {api.bearer}"}, ) deadline: float = time.monotonic() + 30 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py new file mode 100644 index 0000000000..ba69f29812 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py @@ -0,0 +1,204 @@ +"""The real development process tree retains identity, reloads, streams and executes DuckDB work.""" + +import json +import os +import sqlite3 +import subprocess +import sys +import time +from itertools import islice +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DevelopmentTestCase, + SnapshotApi, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + stop_location, + wait_for_generation, + wait_for_log, + wait_for_run, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") +@pytest.mark.parametrize( + "test_case", + [ + DevelopmentTestCase( + "default target and store", + "development", + (), + "project/.sqlbuild/control.sqlite3", + [(1, 5)], + ), + DevelopmentTestCase( + "encoded target and custom store", + "preview #1", + ("--target", "preview #1", "--control-store-url", "{control}"), + "project/custom-control.sqlite3", + [(1, 5)], + '[targets."preview #1"]\nschema = "main"\n', + ), + DevelopmentTestCase( + "local target takes precedence over project default", + "preview #1", + (), + "project/.sqlbuild/control.sqlite3", + [(1, 5)], + 'target = "preview #1"\n[targets."preview #1"]\nschema = "main"\n', + ), + ], + ids=lambda case: case.description, +) +def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paused_defaults_survive( + test_case: DevelopmentTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + pidfile: Path = tmp_path / "interpreter.pid" + declaration: str = ( + "import os\nfrom pathlib import Path\n" + f"Path({str(pidfile)!r}).write_text(str(os.getpid()))\n" + + Path("benchmarks/duckshop/edge_cases/orchestration/development_orders.py").read_text() + ) + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": 'default_target = "development"\n' + + TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '[targets.development]\nschema = "main"\n', + "sqlbuild_local.toml": test_case.local_config, + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "SELECT order_id, amount", "SELECT order_id, CAST(amount AS BIGINT) AS amount" + ), + "orchestration/orders.py": declaration, + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + control: Path = tmp_path / test_case.control_path + arguments: list[str] = [executable, "dev", "--project-dir", str(project), "--port", "0"] + [ + argument.format(control=control) for argument in test_case.arguments + ] + environment: dict[str, str] = os.environ | { + "SQLBUILD_CONTROL_DATABASE_URL": "unsupported://not-development" + } + _ = environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) + log: Path = tmp_path / "development.log" + output: Path = tmp_path / "publications.jsonl" + with location_process(arguments, log, output, env=environment) as process: + wait_for_log(process, log, "Development API ready: ") + url: str = ( + "http://" + log.read_text().partition("Development API ready: ")[2].splitlines()[0] + ) + identity: str = log.read_text().partition("Development location: ")[2].splitlines()[0] + api: SnapshotApi = SnapshotApi(url, identity, "local-development") + scope: str = f"{url}/api/locations/{identity}/targets/{quote(test_case.target, safe='')}" + _ = wait_for_generation(api, test_case.target, 1, process, log) + with urlopen(f"{scope}/triggers", timeout=5) as response: + triggers: list[dict[str, Any]] = json.load(response) + assert len(triggers) == 1 + assert triggers[0]["status"] == "paused" + assert triggers[0]["definition_json"]["default_status"] == "running" + first_interpreter: int = int(pidfile.read_text()) + for headers in [{"Host": "outside.example.com"}, {"Origin": "https://outside.example.com"}]: + with pytest.raises(HTTPError) as rejected: + urlopen(Request(f"{scope}/snapshots", headers=headers), timeout=5) + assert rejected.value.code == 403 + with pytest.raises(HTTPError) as foreign: + urlopen( + f"{url}/api/locations/{uuid4()}/targets/{quote(test_case.target, safe='')}/snapshots", + timeout=5, + ) + assert foreign.value.code == 403 + run_id: str = str(uuid4()) + requested: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "--project-dir", + str(project), + "admin", + "request-run", + identity, + "annual_orders", + "--target", + test_case.target, + "--run-id", + run_id, + "--reason", + "Development execution contract", + ], + env=environment | {"SQLBUILD_CONTROL_DATABASE_URL": str(control)}, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert requested.returncode == 0, requested.stderr + run: dict[str, Any] = wait_for_run( + api, process, log, run_id=run_id, target=test_case.target + ) + assert run["status"] == "success", log.read_text() + time.sleep(1) + assert current_snapshot(api, test_case.target)["record"]["generation"] == 1 + with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: + assert ( + warehouse.sql("SELECT order_id, amount FROM orders").fetchall() + == test_case.expected_rows + ) + stop_location(process, first_interpreter, log) + _ = log.write_text("") + with location_process( + arguments + ["--no-scheduler", "--triggers"], log, output, env=environment + ) as restarted: + wait_for_log(restarted, log, "Development API ready: ") + url = "http://" + log.read_text().partition("Development API ready: ")[2].splitlines()[0] + assert log.read_text().partition("Development location: ")[2].splitlines()[0] == identity + assert "Starting development scheduler" not in log.read_text() + api = SnapshotApi(url, identity, "local-development") + scope = f"{url}/api/locations/{identity}/targets/{quote(test_case.target, safe='')}" + _ = wait_for_generation(api, test_case.target, 2, restarted, log) + assert int(pidfile.read_text()) != first_interpreter + with urlopen(f"{scope}/triggers", timeout=5) as response: + triggers = json.load(response) + assert triggers[0]["status"] == "paused" + with urlopen(f"{scope}/changes", timeout=5) as stream: + assert stream.readline() == b"event: changed\n" + assert json.loads(stream.readline().removeprefix(b"data: ")) == {"topic": "all"} + _ = (project / "orchestration/orders.py").write_text( + declaration + + '\nadditional_orders = job("additional_orders", description="Additional orders.", select="orders", schedule=annual)\n' + ) + assert f"snapshot:{identity}" in islice( + ( + json.loads(line[6:])["topic"] + for line in filter(lambda line: line.startswith(b"data: "), stream) + ), + 100, + ) + _ = wait_for_generation(api, test_case.target, 3, restarted, log) + with urlopen(f"{scope}/triggers", timeout=5) as response: + statuses: dict[str, str] = { + entry["job_key"]: entry["status"] for entry in json.load(response) + } + assert statuses == {"annual_orders": "paused", "additional_orders": "running"} + stop_location(restarted, int(pidfile.read_text()), log) + with sqlite3.connect(control) as connection: + assert connection.execute("SELECT count(*) FROM sqb_development_locations").fetchone() == ( + 1, + ) + for line in output.read_text().splitlines(): + assert json.loads(line)["location_id"] == identity From 34c51c92bbafd0e6b190e47f460413472165ae0d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:50:33 +0000 Subject: [PATCH 063/138] feat: generate and verify the typed control API client Generate TypeScript paths and request/response types from the built native OpenAPI schema. Keep the source-only transport limited to typed requests, with no credential storage, permission fallback or retry policy. Check schema drift and negative typing in CI, and exercise scoped reads, replay, path encoding, anonymous/cross-location denial and SSE against a real native server. --- .github/workflows/verify.yml | 10 + clients/typescript/.gitignore | 1 + clients/typescript/README.md | 40 + clients/typescript/package-lock.json | 431 +++++ clients/typescript/package.json | 20 + clients/typescript/scripts/generate.mjs | 20 + clients/typescript/src/client.ts | 9 + clients/typescript/src/schema.d.ts | 2013 +++++++++++++++++++++++ clients/typescript/tests/client.test.ts | 92 ++ clients/typescript/tests/types.ts | 11 + clients/typescript/tsconfig.json | 13 + 11 files changed, 2660 insertions(+) create mode 100644 clients/typescript/.gitignore create mode 100644 clients/typescript/README.md create mode 100644 clients/typescript/package-lock.json create mode 100644 clients/typescript/package.json create mode 100644 clients/typescript/scripts/generate.mjs create mode 100644 clients/typescript/src/client.ts create mode 100644 clients/typescript/src/schema.d.ts create mode 100644 clients/typescript/tests/client.test.ts create mode 100644 clients/typescript/tests/types.ts create mode 100644 clients/typescript/tsconfig.json diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 73205dc120..410ad4f6c5 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -279,6 +279,16 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: clients/typescript/package-lock.json + - name: Check generated control client against the built native API + run: | + npm --prefix clients/typescript ci + uv run --no-sync npm --prefix clients/typescript run check + uv run --no-sync npm --prefix clients/typescript test - name: Run static checks run: make check-ci - name: Run native checks diff --git a/clients/typescript/.gitignore b/clients/typescript/.gitignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/clients/typescript/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/clients/typescript/README.md b/clients/typescript/README.md new file mode 100644 index 0000000000..8bbaf3f47e --- /dev/null +++ b/clients/typescript/README.md @@ -0,0 +1,40 @@ +# Control API client + +The client uses the native server's OpenAPI request and response types with +`openapi-fetch`. It adds no permissions, credential storage, retries or sync policy. +This package is private and consumed from source; it is not published to npm. + +Rebuild the native package after changing Rust API types, then run from the repository root: + +```sh +uv run --no-sync npm --prefix clients/typescript ci +uv run --no-sync npm --prefix clients/typescript run generate +uv run --no-sync npm --prefix clients/typescript run check +uv run --no-sync npm --prefix clients/typescript test +``` + +Generation invokes `sqb server --openapi` without opening a control database or +importing project Python. Set `SQLBUILD_EXECUTABLE` to test another built binary. +The check regenerates in memory and fails if the committed types differ. Tests use +a real native server, disposable SQLite store and CLI-issued location credential. + +```ts +import { createControlClient } from "@sqlbuild/control-client"; + +const client = createControlClient({ + baseUrl: serverUrl, + headers: { Authorization: `Bearer ${locationCredential}` }, +}); +const { data, error, response } = await client.GET( + "/api/locations/{location_id}/targets/{target}/runs", + { params: { path: { location_id: locationId, target }, query: { limit: 100 } } }, +); +``` + +Inspect `response.status` and `error` before using `data`. Pass an `AbortSignal` +for cancellation. For the scoped changes endpoint, set `parseAs: "stream"` and +consume SSE `changed` frames. An `all` topic, a reconnect, or a lost stream requires +refetching the authorized REST resources; the stream is not durable history. +Browser `EventSource` cannot set a bearer header: use a fetch-based SSE consumer +for the existing machine-authenticated endpoint. Human browser sessions are a +separate authentication contract, not an inferred privilege of this client. diff --git a/clients/typescript/package-lock.json b/clients/typescript/package-lock.json new file mode 100644 index 0000000000..342284a966 --- /dev/null +++ b/clients/typescript/package-lock.json @@ -0,0 +1,431 @@ +{ + "name": "@sqlbuild/control-client", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@sqlbuild/control-client", + "dependencies": { + "openapi-fetch": "0.17.0" + }, + "devDependencies": { + "@types/node": "22.20.5", + "openapi-typescript": "7.13.0", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=22.18.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@redocly/ajv": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", + "integrity": "sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js-replace": "^1.0.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@redocly/config": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/@redocly/config/-/config-0.22.0.tgz", + "integrity": "sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@redocly/openapi-core": { + "version": "1.34.22", + "resolved": "https://registry.npmjs.org/@redocly/openapi-core/-/openapi-core-1.34.22.tgz", + "integrity": "sha512-dEyx4uHh6Pc6ZkdU9Xe8rA4h12Odq6QKU/fqseBn07xwVwskhHiF3KHhqmw5BR+jAiFTvKTmwCafFql/k1pSzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@redocly/ajv": "8.11.2", + "@redocly/config": "0.22.0", + "colorette": "1.4.0", + "https-proxy-agent": "7.0.6", + "js-levenshtein": "1.1.6", + "js-yaml": "4.3.2", + "minimatch": "5.1.9", + "pluralize": "8.0.0", + "yaml-ast-parser": "0.0.43" + }, + "engines": { + "node": ">=18.17.0", + "npm": ">=9.5.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.5.tgz", + "integrity": "sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/change-case": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/change-case/-/change-case-5.4.4.tgz", + "integrity": "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", + "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/index-to-position": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", + "integrity": "sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-levenshtein": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", + "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/openapi-fetch": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.17.0.tgz", + "integrity": "sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==", + "license": "MIT", + "dependencies": { + "openapi-typescript-helpers": "^0.1.0" + } + }, + "node_modules/openapi-typescript": { + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/openapi-typescript/-/openapi-typescript-7.13.0.tgz", + "integrity": "sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@redocly/openapi-core": "^1.34.6", + "ansi-colors": "^4.1.3", + "change-case": "^5.4.4", + "parse-json": "^8.3.0", + "supports-color": "^10.2.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "openapi-typescript": "bin/cli.js" + }, + "peerDependencies": { + "typescript": "^5.x" + } + }, + "node_modules/openapi-typescript-helpers": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.1.0.tgz", + "integrity": "sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==", + "license": "MIT" + }, + "node_modules/parse-json": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", + "integrity": "sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "index-to-position": "^1.1.0", + "type-fest": "^4.39.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/pluralize": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", + "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js-replace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/uri-js-replace/-/uri-js-replace-1.0.2.tgz", + "integrity": "sha512-VeBBt/nMxedczht3KDL2yBBMJqIxpCQLMZp3UJ1otcWwUHdQW4IiyHpJivExzSK+FdxTowRfGL4FmwDqH9+S8Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/yaml-ast-parser": { + "version": "0.0.43", + "resolved": "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz", + "integrity": "sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/clients/typescript/package.json b/clients/typescript/package.json new file mode 100644 index 0000000000..cb648a1b4f --- /dev/null +++ b/clients/typescript/package.json @@ -0,0 +1,20 @@ +{ + "name": "@sqlbuild/control-client", + "private": true, + "type": "module", + "engines": { "node": ">=22.18.0" }, + "exports": "./src/client.ts", + "scripts": { + "generate": "node scripts/generate.mjs", + "check": "node scripts/generate.mjs --check && tsc --noEmit", + "test": "node --experimental-strip-types --test tests/*.test.ts" + }, + "dependencies": { + "openapi-fetch": "0.17.0" + }, + "devDependencies": { + "@types/node": "22.20.5", + "openapi-typescript": "7.13.0", + "typescript": "5.9.3" + } +} diff --git a/clients/typescript/scripts/generate.mjs b/clients/typescript/scripts/generate.mjs new file mode 100644 index 0000000000..07774645ac --- /dev/null +++ b/clients/typescript/scripts/generate.mjs @@ -0,0 +1,20 @@ +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import openapiTS, { astToString } from "openapi-typescript"; + +const destination = new URL("../src/schema.d.ts", import.meta.url); +const document = execFileSync(process.env.SQLBUILD_EXECUTABLE ?? "sqb", ["server", "--openapi"], { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, +}); +const generated = "// Generated from sqb server --openapi. Run npm run generate; do not edit.\n" + + astToString(await openapiTS(JSON.parse(document))); +if (process.argv.includes("--check")) { + if (await readFile(destination, "utf8") !== generated) { + throw new Error("Control client is stale; rebuild sqb and run npm run generate."); + } + console.log("Control client matches the native OpenAPI contract."); +} else { + await writeFile(destination, generated); + console.log("Control client generated from the native OpenAPI contract."); +} diff --git a/clients/typescript/src/client.ts b/clients/typescript/src/client.ts new file mode 100644 index 0000000000..ad9a19251f --- /dev/null +++ b/clients/typescript/src/client.ts @@ -0,0 +1,9 @@ +import createClient, { type ClientOptions } from "openapi-fetch"; +import type { paths } from "./schema.d.ts"; + +export type { components, operations, paths } from "./schema.d.ts"; + +/** Typed transport only: credentials and request cancellation belong to the caller. */ +export function createControlClient(options: ClientOptions) { + return createClient(options); +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts new file mode 100644 index 0000000000..dfa4e2427e --- /dev/null +++ b/clients/typescript/src/schema.d.ts @@ -0,0 +1,2013 @@ +// Generated from sqb server --openapi. Run npm run generate; do not edit. +export interface paths { + "/api/locations/{location_id}/health": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["read_location_health"]; + put?: never; + post: operations["report_location_health"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/changes": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["subscribe"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata/fingerprints": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_fingerprints"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata/source-freshness": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_source_freshness"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["list_runs"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["invocations"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["nodes"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["read_run_recall"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/snapshots": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["current"]; + put?: never; + post: operations["publish"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/snapshots/metadata": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["metadata"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/triggers": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["list_triggers"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["events"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["ticks"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/work/poll": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["poll"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/work/{request_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["result"]; + put?: never; + post: operations["complete"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_run_event"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/execution": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["execution"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/heartbeat": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["heartbeat"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/output": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["output"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/recall": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["recall"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + ApiErrorBody: { + error: string; + }; + CatchUp: "latest" | { + /** @description Process the oldest bounded batch; later due times remain due for the next transaction. */ + all: { + max_runs: number; + }; + }; + /** @description SSE invalidation topics; `all` means refetch the authorized REST resources. */ + ChangeNotice: { + topic: string; + }; + /** @description A row whose key was already stored with different content; the first row is kept. */ + ConflictEvidence: { + existing_sha256: string; + record_key: string; + rejected_sha256: string; + table: components["schemas"]["Table"]; + }; + /** @description The current successful snapshot, read with a consistent generation and change set. */ + CurrentSnapshot: { + changes: components["schemas"]["SnapshotChange"][]; + documents: { + [key: string]: unknown; + }; + nodes: { + [key: string]: components["schemas"]["SnapshotNode"]; + }; + record: components["schemas"]["SnapshotRecord"]; + }; + /** @description Location-owned Python returns data; only the fenced scheduler may apply it. */ + CustomSensorOutcome: { + cursor?: string | null; + run_keys: string[]; + /** @enum {string} */ + status: "completed"; + } | { + cursor?: string | null; + reason: string; + /** @enum {string} */ + status: "skipped"; + } | { + /** @enum {string} */ + status: "failed"; + } | { + /** @enum {string} */ + status: "timed_out"; + }; + /** @description Fingerprint observations from one registered reader, independent of automatic polling. */ + FingerprintBatch: { + /** Format: int64 */ + generation: number; + records: components["schemas"]["WarehouseFingerprintRecord"][]; + /** Format: uuid */ + session_id: string; + }; + HeartbeatResponse: { + cancel_requested: boolean; + }; + /** @description What one insert batch did, per table. */ + InsertOutcome: { + conflicts: components["schemas"]["ConflictEvidence"][]; + /** @description Rows written or already present with equal content, by table. */ + delivered: [ + "invocations" | "output" | "events" | "diagnostics" | "artifacts" | "artifact_refs" | "heartbeats" | "delivery" | "meta" | "conflicts", + number + ][]; + /** @description Tables whose insert failed, so their rows were not written, and the first failure. */ + failed: components["schemas"]["Table"][]; + failure?: string | null; + }; + /** @description Locations report bounded facts, never connection strings or user exception text. */ + LocationHealth: { + /** @enum {string} */ + kind: "heartbeat"; + } | { + /** Format: int64 */ + generation: number; + /** @enum {string} */ + kind: "compile_failed"; + } | { + /** Format: int64 */ + generation: number; + healthy: boolean; + /** @enum {string} */ + kind: "connection"; + }; + /** @description Receipt time belongs to the server; consumers derive freshness rather than storing availability. */ + LocationHealthObservation: { + /** Format: date-time */ + received_at: string; + report: components["schemas"]["LocationHealthReport"]; + }; + LocationHealthReport: { + /** Format: uuid */ + event_id: string; + health: components["schemas"]["LocationHealth"]; + /** Format: uuid */ + session_id: string; + target: string; + }; + /** @description Only named location operations cross this boundary, never arbitrary SQL or Python. */ + LocationWork: { + /** @enum {string} */ + kind: "check_connection"; + } | { + evaluation: Record; + /** @enum {string} */ + kind: "evaluate_sensor"; + }; + LocationWorkClaim: { + /** Format: uuid */ + claim_id: string; + /** Format: int64 */ + generation: number; + /** Format: date-time */ + lease_expires_at: string; + request: components["schemas"]["LocationWorkRequest"]; + /** Format: uuid */ + session_id: string; + }; + LocationWorkOutcome: { + healthy: boolean; + /** @enum {string} */ + kind: "connection"; + } | { + /** @enum {string} */ + kind: "sensor_evaluation"; + outcome: components["schemas"]["CustomSensorOutcome"]; + } | { + /** @enum {string} */ + kind: "superseded"; + }; + LocationWorkPoll: { + /** Format: uuid */ + session_id: string; + target: string; + }; + /** @description Expiry is derived from the deadline; only requests, leases and completed results are stored. */ + LocationWorkRecord: { + /** Format: date-time */ + completed_at?: string | null; + /** Format: date-time */ + expires_at: string; + /** Format: int64 */ + generation: number; + request_json: components["schemas"]["LocationWorkRequest"]; + /** Format: date-time */ + requested_at: string; + requested_by: string; + result_json?: null | components["schemas"]["LocationWorkResult"]; + }; + LocationWorkRequest: { + /** Format: uuid */ + location_id: string; + reason: string; + /** Format: uuid */ + request_id: string; + target: string; + work: components["schemas"]["LocationWork"]; + }; + LocationWorkResult: { + /** Format: uuid */ + claim_id: string; + outcome: components["schemas"]["LocationWorkOutcome"]; + /** Format: uuid */ + session_id: string; + }; + MutationResponse: { + applied: boolean; + }; + /** @enum {string} */ + NodeEventKind: "success"; + /** @enum {string} */ + NodeOutcome: "success" | "warn" | "failed" | "skipped" | "cancelled"; + /** @description Output of one stream that was not recorded because it exceeded `max_output_bytes`. */ + OmittedOutput: { + /** Format: int64 */ + bytes: number; + /** + * Format: int64 + * @description Sequences this gap accounts for, its own included. + */ + sequences: number; + }; + /** @description One exact-byte output chunk. */ + OutputChunk: { + /** + * Format: int64 + * @description Bytes of the same stream before this chunk. + */ + byte_offset: number; + data: number[]; + /** + * Format: int64 + * @description Output records the queue dropped just before this one. + */ + dropped_before: number; + occurred_at: string; + omitted?: null | components["schemas"]["OmittedOutput"]; + /** + * Format: int64 + * @description Per invocation, from 0, shared by both streams. + */ + sequence: number; + stream: components["schemas"]["OutputStream"]; + }; + /** @description Differing retries retain conflict evidence while preserving the first output bytes. */ + OutputPublication: { + /** Format: int32 */ + conflicts: number; + /** Format: int32 */ + duplicates: number; + /** Format: int32 */ + inserted: number; + }; + /** + * @description A process output stream. + * @enum {string} + */ + OutputStream: "stdout" | "stderr"; + /** @enum {string} */ + Overlap: "coalesce" | "queue" | "allow"; + /** @enum {string} */ + PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; + /** @description One invocation's records; the server owns its claim namespace, start time and version. */ + RecallBatch: { + invocation_id: string; + records: components["schemas"]["RecallRecord"][]; + }; + RecallRecord: { + table: components["schemas"]["Table"]; + values: { + [key: string]: components["schemas"]["RecallValue"]; + }; + }; + /** @description One stored value. */ + RecallValue: { + /** @enum {string} */ + kind: "null"; + } | { + /** @enum {string} */ + kind: "integer"; + /** Format: int64 */ + value: number; + } | { + /** @enum {string} */ + kind: "real"; + /** Format: double */ + value: number; + } | { + /** @enum {string} */ + kind: "text"; + value: string; + } | { + /** @enum {string} */ + kind: "blob"; + value: number[]; + }; + /** @description This is a server-owned instruction, not an arbitrary status supplied by a worker. */ + RunChange: { + container_id?: string | null; + /** @enum {string} */ + kind: "launched"; + } | { + code: string; + /** @enum {string} */ + kind: "launch_failed"; + } | { + /** @enum {string} */ + kind: "started"; + } | { + /** @enum {string} */ + kind: "node_started"; + node_key: string; + } | { + error_code?: string | null; + /** @enum {string} */ + kind: "node_finished"; + node_key: string; + outcome: components["schemas"]["NodeOutcome"]; + /** Format: int64 */ + rows_affected?: number | null; + skipped_because_of?: string | null; + } | { + /** @enum {string} */ + kind: "unresponsive"; + } | { + /** @enum {string} */ + kind: "recovered"; + } | { + actor: string; + /** @enum {string} */ + kind: "cancel_requested"; + reason?: string; + } | { + /** Format: int64 */ + exit_code?: number | null; + /** @enum {string} */ + kind: "finished"; + outcome: components["schemas"]["RunOutcome"]; + reason: string; + }; + RunChangeRequest: { + change: components["schemas"]["RunChange"]; + /** Format: uuid */ + claim_id?: string | null; + /** Format: uuid */ + event_id: string; + /** Format: uuid */ + run_id: string; + }; + /** @description The database owns the claim before a launcher sees the pinned request. */ + RunClaim: { + /** Format: uuid */ + claim_id: string; + request: components["schemas"]["RunRequest"]; + }; + /** @enum {string} */ + RunEventKind: "completed" | "success" | "warn" | "partial" | "failed" | "cancelled" | "presumed_failed"; + /** @description Workers compile locally and compare against this immutable snapshot, not the current deploy. */ + RunExecution: { + claim: components["schemas"]["RunClaim"]; + environment_hash: string; + node_hashes: { + [key: string]: string; + }; + /** @description Derived from completed sibling facts and pinned lineage, never a mutable request rewrite. */ + skipped_nodes?: { + [key: string]: string; + }; + sqb_version: string; + }; + /** @enum {string} */ + RunKind: "job" | "group" | "adhoc"; + RunNodeRecord: { + /** Format: int64 */ + attempt: number; + /** Format: date-time */ + ended_at?: string | null; + error_code?: string | null; + node_key: string; + /** Format: int64 */ + rows_affected?: number | null; + skipped_because_of?: string | null; + /** Format: date-time */ + started_at?: string | null; + status: string; + }; + /** @enum {string} */ + RunOrigin: "manual" | "schedule" | "sensor" | "rerun" | "backfill"; + /** + * @description Terminal outcomes cannot be confused with a request to change an active status. + * @enum {string} + */ + RunOutcome: "success" | "warn" | "partial" | "failed" | "cancelled" | "presumed_failed"; + RunRecord: { + /** Format: date-time */ + ended_at?: string | null; + job_key?: string | null; + /** Format: int64 */ + last_event_storage_id: number; + /** Format: date-time */ + launched_at?: string | null; + /** Format: uuid */ + location_id: string; + node_counts_json: { + [key: string]: number; + }; + /** Format: date-time */ + queued_at: string; + /** Format: uuid */ + run_id: string; + selection_json: string[]; + /** Format: uuid */ + snapshot_id: string; + /** Format: date-time */ + started_at?: string | null; + status: components["schemas"]["RunStatus"]; + target: string; + }; + /** @description The authorized selection and pinned execution inputs are immutable once requested. */ + RunRequest: { + concurrency_groups: string[]; + image_id?: string | null; + job_key?: string | null; + kind: components["schemas"]["RunKind"]; + /** Format: uuid */ + location_id: string; + origin: components["schemas"]["RunOrigin"]; + overlap: components["schemas"]["Overlap"]; + /** Format: uuid */ + parent_run_id?: string | null; + /** Format: int64 */ + priority: number; + requested_by: string; + /** Format: uuid */ + rerun_of_run_id?: string | null; + /** Format: uuid */ + root_run_id: string; + /** Format: uuid */ + run_id: string; + selection: string[]; + settings: { + [key: string]: unknown; + }; + /** Format: uuid */ + snapshot_id: string; + target: string; + /** Format: uuid */ + tick_id?: string | null; + /** Format: uuid */ + trigger_id?: string | null; + window?: null | components["schemas"]["RunWindow"]; + write_set: string[]; + }; + /** @enum {string} */ + RunStatus: "queued" | "launched" | "running" | "unresponsive" | "success" | "warn" | "partial" | "failed" | "cancelled" | "presumed_failed"; + /** @description Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. */ + RunWindow: { + /** Format: date-time */ + end: string; + /** @enum {string} */ + kind: "timestamp"; + /** Format: date-time */ + start: string; + } | { + /** Format: int64 */ + end: number; + /** @enum {string} */ + kind: "integer"; + /** Format: int64 */ + start: number; + }; + Schedule: { + expression: string; + /** @enum {string} */ + kind: "cron"; + timezone: string; + } | { + /** @enum {string} */ + kind: "calendar"; + /** Format: date-time */ + refresh_at: string; + times: string[]; + }; + SensorCondition: { + /** @enum {string} */ + kind: "platform_event"; + kinds: components["schemas"]["PlatformEventKind"][]; + } | { + executed_successfully?: string[] | null; + job_key: string; + /** @enum {string} */ + kind: "run_event"; + kinds: components["schemas"]["RunEventKind"][]; + } | { + /** @enum {string} */ + kind: "node_event"; + kinds: components["schemas"]["NodeEventKind"][]; + selection: string[]; + } | { + code_hash: string; + function: string; + /** @enum {string} */ + kind: "custom"; + }; + SensorDefinition: { + default_status?: components["schemas"]["SensorStatus"]; + description: string; + job_key: string; + /** Format: int32 */ + minimum_interval_seconds: number; + when: components["schemas"]["SensorCondition"]; + }; + /** @enum {string} */ + SensorStatus: "running" | "stopped"; + /** @description Node-level changes do not duplicate unchanged definitions or source text. */ + SnapshotChange: { + after_hash?: string | null; + before_hash?: string | null; + change_kind: string; + subject_key: string; + }; + /** @description A location's successful compile for one target. */ + SnapshotInput: { + documents: { + [key: string]: unknown; + }; + environment_hash: string; + /** Format: int64 */ + generation: number; + git_author?: string | null; + git_dirty: boolean; + git_revision?: string | null; + image_id?: string | null; + /** Format: uuid */ + location_id: string; + nodes: { + [key: string]: components["schemas"]["SnapshotNode"]; + }; + sqb_version: string; + target: string; + }; + /** @description A compiled node and its display-only source. SQL and Python are never run by the server. */ + SnapshotNode: { + compiled_sql?: string | null; + definition: unknown; + kind: string; + python_source?: string | null; + raw_sql?: string | null; + }; + /** @description Immutable publication metadata. Generations increase independently per location and target. */ + SnapshotRecord: { + /** Format: int64 */ + generation: number; + /** Format: uuid */ + location_id: string; + manifest_digest: string; + /** Format: date-time */ + published_at: string; + /** Format: uuid */ + snapshot_id: string; + target: string; + }; + /** @description A bounded page from one registered reader of a published location/target generation. */ + SourceFreshnessBatch: { + /** Format: int64 */ + generation: number; + records: components["schemas"]["WarehouseFreshnessRecord"][]; + /** Format: uuid */ + session_id: string; + }; + /** + * @description The tables Recall writes. + * @enum {string} + */ + Table: "invocations" | "output" | "events" | "diagnostics" | "artifacts" | "artifact_refs" | "heartbeats" | "delivery" | "meta" | "conflicts"; + /** @enum {string} */ + TickOutcome: "requested" | "skipped" | "covered" | "failed" | "deferred"; + TickRecord: { + /** Format: uuid */ + covered_by_run_id?: string | null; + /** Format: int64 */ + epoch: number; + /** Format: date-time */ + evaluated_at: string; + outcome: components["schemas"]["TickOutcome"]; + reason?: string | null; + /** Format: uuid */ + run_id?: string | null; + /** Format: date-time */ + scheduled_for: string; + /** Format: uuid */ + tick_id: string; + /** Format: uuid */ + trigger_id: string; + }; + /** @description Operator changes are immutable facts; clearing an override restores the published code default. */ + TriggerChange: { + /** @enum {string} */ + kind: "enable"; + } | { + /** @enum {string} */ + kind: "pause"; + } | { + /** @enum {string} */ + kind: "reset_default"; + } | { + /** @enum {string} */ + kind: "set_cursor"; + /** Format: date-time */ + time: string; + } | { + /** @enum {string} */ + kind: "reset_cursor"; + }; + TriggerChangeRecord: { + /** Format: date-time */ + occurred_at: string; + request_json: components["schemas"]["TriggerChangeRequest"]; + /** Format: uuid */ + snapshot_id: string; + }; + TriggerChangeRequest: { + actor: string; + change: components["schemas"]["TriggerChange"]; + /** Format: uuid */ + event_id: string; + reason: string; + /** Format: uuid */ + trigger_id: string; + }; + /** @description Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. */ + TriggerDeclaration: components["schemas"]["TriggerDefinition"] | components["schemas"]["SensorDefinition"]; + TriggerDefinition: { + catch_up?: components["schemas"]["CatchUp"]; + default_status?: components["schemas"]["TriggerStatus"]; + description: string; + job_key: string; + schedule: components["schemas"]["Schedule"]; + /** Format: date-time */ + starts_at?: string | null; + }; + TriggerRecord: { + definition_json: components["schemas"]["TriggerDeclaration"]; + job_key: string; + /** Format: date-time */ + last_tick_for?: string | null; + name: string; + /** Format: date-time */ + next_due_at?: string | null; + status: components["schemas"]["TriggerStatus"]; + /** Format: uuid */ + trigger_id: string; + }; + /** @enum {string} */ + TriggerStatus: "running" | "paused"; + /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ + WarehouseFingerprintRecord: { + definition: string; + definition_hash: string; + event_id?: string | null; + metadata_json: string; + node_name: string; + node_type: string; + run_id: string; + schema_fingerprint: string; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + /** Format: date-time */ + ts: string; + version_hash: string; + }; + /** @description Preserve observations, not an inferred latest version, age status or run outcome. */ + WarehouseFreshnessRecord: { + data_version?: string | null; + data_version_hash: string; + event_id?: string | null; + /** Format: date-time */ + observed_at: string; + run_id: string; + source_name: string; + state_database?: string | null; + state_schema: string; + strategy: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + value_kind: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + read_location_health: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["LocationHealthObservation"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_location_health: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["LocationHealthReport"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + subscribe: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description changed events: all, runs, run:, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/event-stream": components["schemas"]["ChangeNotice"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_fingerprints: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["FingerprintBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_source_freshness: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SourceFreshnessBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + list_runs: { + parameters: { + query?: { + /** @description Maximum recent runs; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + invocations: { + parameters: { + query?: { + /** @description Maximum invocation namespaces; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown[]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + nodes: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunNodeRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + read_run_recall: { + parameters: { + query: { + /** @description Exact scheduler or worker invocation ID */ + invocation: string; + /** @description Maximum rows; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + view: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown[]; + }; + }; + /** @description Unknown Recall view */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run or invocation not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + current: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CurrentSnapshot"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description No successful snapshot */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + publish: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SnapshotInput"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SnapshotRecord"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Stale generation */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + metadata: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SnapshotRecord"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description No successful snapshot */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + list_triggers: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TriggerRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + events: { + parameters: { + query?: { + /** @description Maximum events; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + trigger_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TriggerChangeRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Trigger not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + ticks: { + parameters: { + query?: { + /** @description Maximum recent ticks; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + trigger_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TickRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Trigger not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + poll: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["LocationWorkPoll"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkClaim"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + result: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkRecord"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + complete: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + request_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["LocationWorkResult"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Work claim expired or was replaced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_run_event: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RunChangeRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + execution: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunExecution"]; + }; + }; + /** @description Inactive claim or invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + heartbeat: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HeartbeatResponse"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + output: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["OutputChunk"][]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OutputPublication"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + recall: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RecallBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["InsertOutcome"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; +} diff --git a/clients/typescript/tests/client.test.ts b/clients/typescript/tests/client.test.ts new file mode 100644 index 0000000000..7385be356d --- /dev/null +++ b/clients/typescript/tests/client.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { once } from "node:events"; +import { test } from "node:test"; +import { createControlClient, type components } from "../src/client.ts"; + +test("generated client preserves native HTTP scopes, replay, path encoding and SSE", { timeout: 20000 }, async () => { + const directory = await mkdtemp(join(tmpdir(), "sqlbuild-client-")); + const executable = process.env.SQLBUILD_EXECUTABLE ?? "sqb"; + const env = { ...process.env }; + delete env.SQLBUILD_CONTROL_DATABASE_URL; + delete env.SQLBUILD_PLATFORM_WEBHOOK_URL; + const admin = (...args: string[]) => JSON.parse(execFileSync(executable, ["admin", ...args], { + cwd: directory, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], + })); + let server: ReturnType | undefined; + try { + const { location_id } = admin("create-location", "orders"); + const other = admin("create-location", "inventory").location_id; + const { bearer } = admin("issue-location-token", location_id, "--expires-at", + new Date(Date.now() + 60000).toISOString()); + server = spawn(executable, ["server", "--port", "0", "--no-scheduler"], { + cwd: directory, env, stdio: ["ignore", "ignore", "pipe"], + }); + const running = server; + // Readiness is emitted on stderr; never log the one-time credential response. + const baseUrl = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("API readiness timed out")), 5000); + let log = ""; + running.once("error", reject); + running.once("exit", () => { clearTimeout(timer); reject(new Error("API exited before readiness")); }); + running.stderr!.on("data", (chunk: Buffer) => { + log += chunk.toString(); + const address = /Control API ready: ([^\r\n]+)/.exec(log); + if (address) { clearTimeout(timer); resolve(`http://${address[1]}`); } + }); + }); + const client = createControlClient({ baseUrl, headers: { Authorization: `Bearer ${bearer}` } }); + const healthPath = "/api/locations/{location_id}/health"; + const params = { path: { location_id } }; + const body: components["schemas"]["LocationHealthReport"] = { + event_id: randomUUID(), session_id: randomUUID(), target: "preview #1", health: { kind: "heartbeat" }, + }; + const written = await client.POST(healthPath, { params, body }); + assert.equal(written.response.status, 200); + assert.deepEqual(written.data, { applied: true }); + const replay = await client.POST(healthPath, { params, body }); + assert.deepEqual(replay.data, { applied: false }); + const read = await client.GET(healthPath, { params }); + assert.equal(read.response.headers.get("cache-control"), "no-store"); + assert.deepEqual(read.data?.map((row) => row.report), [body]); + const forbidden = await client.GET(healthPath, { params: { path: { location_id: other } } }); + assert.equal(forbidden.response.status, 403); + assert.equal(forbidden.data, undefined); + const anonymous = await createControlClient({ baseUrl }).GET(healthPath, { params }); + assert.equal(anonymous.response.status, 401); + const runs = await client.GET("/api/locations/{location_id}/targets/{target}/runs", { + params: { path: { location_id, target: "preview #1" }, query: { limit: 1 } }, + }); + assert.equal(runs.response.status, 200); + assert.deepEqual(runs.data, []); + const changes = await client.GET("/api/locations/{location_id}/targets/{target}/changes", { + params: { path: { location_id, target: "preview #1" } }, parseAs: "stream", + signal: AbortSignal.timeout(3000), + }); + assert.equal(changes.response.headers.get("content-type"), "text/event-stream"); + const reader = changes.data!.pipeThrough(new TextDecoderStream()).getReader(); + try { + let event = ""; + while (!event.includes("\n\n")) { + const chunk = await reader.read(); + assert.equal(chunk.done, false); + event += chunk.value; + } + assert.match(event, /event: changed/); + assert.match(event, /data: \{"topic":"all"\}/); + } finally { + await reader.cancel(); + } + } finally { + if (server && server.exitCode === null) { + const exited = once(server, "exit"); + server.kill("SIGTERM"); + await exited; + } + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts new file mode 100644 index 0000000000..5ad1a573b7 --- /dev/null +++ b/clients/typescript/tests/types.ts @@ -0,0 +1,11 @@ +import { createControlClient, type components } from "../src/client.ts"; + +// Compile-only consumer contracts: these must fail if generated typing is lost. +const client = createControlClient({ baseUrl: "https://example.com" }); +// @ts-expect-error Unknown paths are not accepted. +void client.GET("/api/not-a-route"); +// @ts-expect-error The location path parameter is required. +void client.GET("/api/locations/{location_id}/health", {}); +// @ts-expect-error Only the native discriminated health variants are accepted. +const invalid: components["schemas"]["LocationHealth"] = { kind: "unknown" }; +void invalid; diff --git a/clients/typescript/tsconfig.json b/clients/typescript/tsconfig.json new file mode 100644 index 0000000000..2a2da4290f --- /dev/null +++ b/clients/typescript/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "types": ["node"] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} From 29420fd040ec9e09713df671509aa6f819eb90d4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:26:45 +0000 Subject: [PATCH 064/138] feat: resolve independent named catalog connections --- .../orchestration/catalog_connection.toml | 18 ++++++ crates/README.md | 8 ++- .../src/effective/constants.rs | 7 +++ .../src/project/_helpers/targets.rs | 4 ++ .../sqlbuild-config/src/project/constants.rs | 3 +- .../src/project/tests/test_load_config.rs | 17 ++++++ .../_helpers/command_runtime/connection.rs | 4 ++ .../compile_attachment/effective_config.rs | 31 ++++++++-- .../_helpers/compile_driver/pipeline.rs | 1 + .../_helpers/compile_driver/targets.rs | 2 +- .../_helpers/contract_records/constants.rs | 2 + .../_helpers/contracts/effective_config.rs | 44 +++++++++++++-- .../_helpers/dbt_interop/plan_output.rs | 2 +- .../_helpers/debug_command/run_debug.rs | 1 + .../bindings/_helpers/load/load_selection.rs | 1 + .../native_host/warehouse_metadata.rs | 1 + src/sqlbuild/_native.pyi | 11 ++++ .../publication/_test_types.py | 8 +++ .../publication/test_warehouse_metadata.py | 42 ++++++++++++++ .../_helpers/runtime/test_connection.py | 56 +++++++++++++++++++ 20 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml diff --git a/benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml b/benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml new file mode 100644 index 0000000000..c610cac914 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml @@ -0,0 +1,18 @@ +name = "orders" +adapter = "sqlite" +default_target = "development" + +[connection] +path = "must-not-open.db" +password = "${ENV:SQLBUILD_MISSING_WRITER_PASSWORD}" + +[connections.writer] +path = "must-not-open-writer.db" + +[connections.catalog] +path = "${ENV:SQLBUILD_CATALOG_DATABASE}" + +[targets.development] +connection = "writer" +catalog_connection = "catalog" +schema = "main" diff --git a/crates/README.md b/crates/README.md index 8307293494..28ea346d8b 100644 --- a/crates/README.md +++ b/crates/README.md @@ -167,7 +167,13 @@ and future storage columns do not alter these identities. The location history reader pages fingerprint and source-freshness observations without selecting only the latest row. It is read-only, including local database connections: it neither creates -databases nor evolves tables. Fingerprint definitions and metadata use the existing storage decoder; +databases nor evolves tables. A target can select `catalog_connection = "catalog"` from named +connections; project and local settings for that name resolve independently, with normal secret +expansion and adapter defaults. Project, local and target writer credentials are never overlaid or +used to fill missing catalog credentials. Without the override, reads use the target's normal +connection. Local target overrides select the catalog name through the normal target precedence; +unknown names and blank or non-string selections are configuration errors. +Fingerprint definitions and metadata use the existing storage decoder; raw storage values remain the keyset cursor, separate from decoded UTC timestamps and text. Fingerprint and source-freshness ingestion preserve warehouse IDs, scoped by location, target and physical state table. Only missing or NULL IDs use fixed legacy fields, never a hash of an evolving diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index b7a0f2b172..f89d6956c6 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -217,4 +217,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["concurrency"], }, }, + TargetField { + group: None, + name: "catalog_connection", + rule: MergeRule::Overlay { + decided_by: &["catalog_connection"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index 0ba2fcebd1..1d0f4579c0 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -415,6 +415,10 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded>()?; fields.push(("clone", object("LocalClonePolicy", clone_fields))); fields.push(("concurrency", target_concurrency(mapping, target)?)); + fields.push(( + "catalog_connection", + optional_text_field(mapping, "catalog_connection")?, + )); Ok(fields) } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 7b56d857d7..c3ae524611 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -172,8 +172,9 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "valid_from_column", "valid_to_column", ]; -pub(crate) const TARGET_KEYS: [&str; 20] = [ +pub(crate) const TARGET_KEYS: [&str; 21] = [ "connection", + "catalog_connection", "vars", "database", "schema", diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 2479ee5592..e556567d29 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,3 +1,4 @@ +use crate::project::errors::ConfigFailure; use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; @@ -19,6 +20,22 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { local: "target = 'dev'\n[settings]\nmax_concurrency = 2\n", expected_failure: None, }, + ProjectFailureTestCase { + description: "a blank catalog connection is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[targets.dev]\ncatalog_connection = ' '\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'catalog_connection' to be a non-empty string when provided", + )), + }, + ProjectFailureTestCase { + description: "a catalog connection must be a name, not credential overrides", + project: "name = 'orders'\nadapter = 'duckdb'\n", + local: "[targets.dev]\ncatalog_connection = { user = 'catalog-reader' }\n", + expected_failure: Some(ConfigFailure::plain( + "Expected 'catalog_connection' to be a non-empty string when provided", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 0f6a0d3127..495410b7a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -366,6 +366,7 @@ pub(crate) fn project_connection_config<'py>( discovered_inputs: &Bound<'py, PyAny>, (project_dir, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (cli_vars, require_connection): (&Bound<'py, PyAny>, bool), + catalog: bool, ) -> PyResult> { if require_connection { require_configured_connection(discovered_inputs, project_dir, selected_target)?; @@ -374,6 +375,7 @@ pub(crate) fn project_connection_config<'py>( &discovered_inputs.clone(), Some(selected_target.clone()).filter(|value| !value.is_none()), Some(cli_vars.clone()).filter(|value| !value.is_none()), + catalog, )? .cast_into()?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( @@ -410,6 +412,7 @@ pub(crate) fn resolve_project_connection_config<'py>( &arguments.value("selected_target")?, ), (&arguments.value("cli_vars")?, require_connection), + false, ) } @@ -433,6 +436,7 @@ pub(crate) fn resolve_target_connection_config<'py>( &arguments.value("target_name")?, ), (&arguments.value("cli_vars")?, true), + false, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index 382aa2f4bc..e1936e7e25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -7,9 +7,9 @@ use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::contracts::effective_config::effective_config_connection; use crate::bindings::_helpers::contracts::effective_config::effective_config_settings; use crate::bindings::_helpers::contracts::effective_config::effective_config_vars; +use crate::bindings::_helpers::contracts::effective_config::effective_connection_fields; use crate::bindings::_helpers::contracts::spec_resolution::{ resolve_target_config, resolve_target_name, }; @@ -31,10 +31,25 @@ pub(crate) fn build_effective_connection<'py>( local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + expanded_connection( + (project_config, local_config), + target_config, + effective_vars, + false, + ) +} + +/// Expand only the connection selected for the caller's writer or catalog role. +fn expanded_connection<'py>( + (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + catalog: bool, ) -> PyResult> { let py: Python<'py> = project_config.py(); let (connection, error): (Bound<'py, PyAny>, Option) = - effective_config_connection(py, project_config, local_config, target_config) + effective_connection_fields(py, (project_config, local_config), target_config, catalog) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? .extract()?; if connection.is_none() { @@ -156,20 +171,24 @@ pub(crate) fn build_effective_target_config<'py>( /// Build the effective project connection config without compiling resources. #[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None, catalog = false))] pub(crate) fn build_effective_connection_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, cli_vars: Option>, + catalog: bool, ) -> PyResult> { let (_, target_config) = target_selection(discovered_inputs, selected_target)?; let effective_vars: Bound<'py, PyAny> = discovered_vars(discovered_inputs, &target_config, cli_vars)?; - build_effective_connection( - &native_field(discovered_inputs, "project_config")?, - &native_field(discovered_inputs, "local_config")?, + expanded_connection( + ( + &native_field(discovered_inputs, "project_config")?, + &native_field(discovered_inputs, "local_config")?, + ), &target_config, &effective_vars, + catalog, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index ddcf1ae6dd..689bbfafe6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -75,6 +75,7 @@ pub(crate) fn compiled_phase<'py>( discovered, Some(native_field(&options, "selected_target")?), Some(native_field(&options, "cli_vars")?), + false, )? } else { configured diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index dd9f0a762b..41b587e0f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -274,7 +274,7 @@ fn target_namespaces( let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = runtime.extract()?; let connection: Bound<'_, PyAny> = if resolved_connection.is_none() { - build_effective_connection_config(&discovered.clone(), Some(name), None) + build_effective_connection_config(&discovered.clone(), Some(name), None, false) .and_then(|value| value.into_bound_py_any(py))? } else { resolved_connection.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 34186a05d0..8a268be998 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -147,6 +147,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("catalog_connection", FieldDefault::None), ], }; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { @@ -186,6 +187,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("catalog_connection", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs index 896dcdd112..6a4176ae83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs @@ -287,7 +287,25 @@ pub(crate) fn effective_config_connection<'py>( local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, ) -> PyResult>> { - let target: Option<(Option>, Pairs<'py>)> = if target_config.is_none() { + effective_connection_fields(py, (project_config, local_config), target_config, false) +} + +/// Catalog selection uses the same named-layer resolver, with no writer layers. +pub(crate) fn effective_connection_fields<'py>( + py: Python<'py>, + (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + catalog: bool, +) -> PyResult>> { + let catalog_name = if catalog && !target_config.is_none() { + optional_key(&native_field(target_config, "catalog_connection")?)? + } else { + None + }; + let independent = catalog_name.is_some(); + let target: Option<(Option>, Pairs<'py>)> = if independent { + Some((catalog_name, Vec::new())) + } else if target_config.is_none() { None } else { Some(( @@ -296,15 +314,33 @@ pub(crate) fn effective_config_connection<'py>( )) }; let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { - project: pairs(&native_field(project_config, "connection")?)?, + project: if independent { + Vec::new() + } else { + pairs(&native_field(project_config, "connection")?)? + }, target, project_named: named(&native_field(project_config, "connections")?)?, local_named: named(&native_field(local_config, "connections")?)?, - local: pairs(&native_field(local_config, "connection")?)?, + local: if independent { + Vec::new() + } else { + pairs(&native_field(local_config, "connection")?)? + }, }; Ok(match effective_connection(layers) { Ok(connection) => (Some(dictionary(py, connection)?), None), - Err(error) => (None, Some(message(py, &error)?)), + Err(error) => { + let message = message(py, &error)?; + ( + None, + Some(if independent { + PyString::new(py, &format!("catalog_connection: {message}")) + } else { + message + }), + ) + } }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 4b3fdea078..d140a8017d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -42,7 +42,7 @@ pub(crate) fn connection_config<'py>( ) -> PyResult> { let py = environment.py(); let discovered = native_field(environment, "discovered_inputs")?; - let raw = build_effective_connection_config(&discovered.clone(), None, None)?; + let raw = build_effective_connection_config(&discovered.clone(), None, None, false)?; let raw: Bound<'_, PyDict> = raw.cast_into()?; let project_dir = native_field(environment, "project_dir")?; let adapter_name = native_field(environment, "adapter_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index c0e0910921..0d8524bc71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -293,6 +293,7 @@ fn build_report( &context.discovered_inputs, (&context.project_dir, &context.selected_target), (&py.None().into_bound(py), true), + false, ) })?; let connection_config: Bound<'_, PyDict> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 9ee1fcb3d2..b20cc9d2a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -260,6 +260,7 @@ pub(crate) fn loader_defaults<'py>( &discovered.clone(), Some(selected_target.clone()).filter(|value| !value.is_none()), Some(cli_vars.clone()).filter(|value| !value.is_none()), + false, )? .downcast_into()?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index d07490aeb7..e0da01f39b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -59,6 +59,7 @@ pub(crate) fn read_history( &discovered, (&project_dir, &arguments.value("selected_target")?), (&py.None().into_bound(py), true), + true, )?; let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; if let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 41dade4272..52f5fbd788 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2352,6 +2352,13 @@ def resolve_target_connection_config( target_name: str, cli_vars: dict[str, object] | None = None, ) -> dict[str, object]: ... +def build_effective_connection_config( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, + catalog: bool = False, +) -> dict[str, object]: ... def normalize_dbt_profile_connection( *, resolved: ResolvedDbtProfileOutput ) -> NormalizedDbtProfileConnection: ... @@ -11809,6 +11816,7 @@ class TargetConfig(NativeRecord): diff: TargetDiffConfig warehouses: TargetWarehousesConfig concurrency: int | None + catalog_connection: str | None def __init__( self, connection: dict[str, object] = ..., @@ -11832,6 +11840,7 @@ class TargetConfig(NativeRecord): diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., concurrency: int | None = ..., + catalog_connection: str | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -11858,6 +11867,7 @@ class LocalTargetConfig(NativeRecord): diff: TargetDiffConfig warehouses: TargetWarehousesConfig concurrency: int | None + catalog_connection: str | None def __init__( self, connection: dict[str, object] = ..., @@ -11882,6 +11892,7 @@ class LocalTargetConfig(NativeRecord): diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., concurrency: int | None = ..., + catalog_connection: str | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index efb06e04fa..2dfad6646e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -157,6 +157,14 @@ class WarehouseHistoryTestCase: append_fingerprints: Callable[..., None] +@dataclass(frozen=True) +class CatalogConnectionTestCase: + description: str + adapter_name: str + connection_key: str + expected_run_id: str + + @dataclass(frozen=True) class FingerprintHistoryTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 2b328d4c5f..440f4f3bad 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -20,6 +20,7 @@ write_source_freshness_records, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CatalogConnectionTestCase, FingerprintHistoryTestCase, WarehouseHistoryTestCase, ) @@ -31,6 +32,47 @@ ) +@pytest.mark.parametrize( + "test_case", + [ + CatalogConnectionTestCase("DuckDB independent reader", "duckdb", "database", "legacy-run"), + CatalogConnectionTestCase("SQLite independent reader", "sqlite", "path", "legacy-run"), + ], + ids=lambda case: case.description, +) +def test_given_independent_catalog_connection_when_reading_history_then_writer_secrets_and_files_are_untouched( + test_case: CatalogConnectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + connection_key: str = test_case.connection_key + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, test_case.adapter_name, connection_key) + adapter.close(connection) + config: Path = project / "sqlbuild_project.toml" + config.write_text( + Path("benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml") + .read_text() + .replace('adapter = "sqlite"', f'adapter = "{test_case.adapter_name}"') + .replace("\npath =", f"\n{connection_key} =") + ) + monkeypatch.delenv("SQLBUILD_MISSING_WRITER_PASSWORD", raising=False) + monkeypatch.setenv("SQLBUILD_CATALOG_DATABASE", "warehouse.db") + first: Any = warehouse_history_page(project, "source_freshness") + assert first is not None and len(first["records"]) == 1 + assert first["records"][0]["run_id"] == test_case.expected_run_id + # Local target selection replaces only the catalog name, not the writer connection. + (project / "warehouse.db").rename(project / "retained-warehouse.db") + (project / "sqlbuild_local.toml").write_text( + f'[connections.mirror]\n{connection_key} = "retained-warehouse.db"\n' + '[targets.development]\ncatalog_connection = "mirror"\n' + ) + assert warehouse_history_page(project, "source_freshness") == first + assert not (project / "must-not-open.db").exists() + assert not (project / "must-not-open-writer.db").exists() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index fd1642cba4..f8964da4f8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -11,6 +11,7 @@ ProjectConfig, ProjectConfigError, TargetConfig, + build_effective_connection_config, resolve_connection_config, resolve_project_connection_config, resolve_target_connection_config, @@ -174,6 +175,61 @@ def test_given_named_connection_layers_when_resolving_then_uses_locked_precedenc assert connection == test_case.expected_connection +@pytest.mark.parametrize( + "test_case", + [ + NamedConnectionBehaviorTestCase( + description="catalog credentials never inherit writer fields", + expected_connection={ + "account": "catalog-account", + "user": "catalog-reader", + "warehouse": "catalog-warehouse", + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_catalog_connection_when_resolving_then_only_its_own_named_layers_and_secrets_apply( + test_case: NamedConnectionBehaviorTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_MISSING_WRITER_PASSWORD", raising=False) + monkeypatch.setenv("SQLBUILD_CATALOG_USER", "catalog-reader") + discovered: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig( + name="orders", + adapter="snowflake", + default_target="development", + connection={"password": "${ENV:SQLBUILD_MISSING_WRITER_PASSWORD}"}, + connections={ + "writer": {"account": "writer-account", "private_key_file": "writer.pem"}, + "catalog": {"account": "catalog-account", "user": "${ENV:SQLBUILD_CATALOG_USER}"}, + }, + targets={ + "development": TargetConfig( + connection_name="writer", + catalog_connection="catalog", + connection={"role": "writer-role"}, + ) + }, + ), + local_config=LocalConfig( + connection={"token": "writer-token"}, + connections={"catalog": {"warehouse": "catalog-warehouse"}}, + targets={"development": LocalTargetConfig(connection={"user": "writer-user"})}, + ), + ) + assert ( + build_effective_connection_config(discovered_inputs=discovered, catalog=True) + == test_case.expected_connection + ) + with pytest.raises(CompileInputError, match="SQLBUILD_MISSING_WRITER_PASSWORD"): + build_effective_connection_config(discovered_inputs=discovered) + discovered.local_config.targets["development"] = LocalTargetConfig(catalog_connection="missing") + with pytest.raises(CompileInputError, match="catalog_connection: Unknown connection 'missing'"): + build_effective_connection_config(discovered_inputs=discovered, catalog=True) + + @pytest.mark.parametrize( "test_case", [ From 9c62bd3fde73fa7ddabb398bb957d7ad6b1859af Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:51:55 +0000 Subject: [PATCH 065/138] feat: derive individual failure and recovery transitions --- .../orchestration/check_transitions.json | 13 ++ clients/typescript/src/schema.d.ts | 2 +- crates/README.md | 17 ++ .../src/sensors/_helpers/conditions.rs | 4 +- .../src/sensors/_helpers/mod.rs | 1 + .../src/sensors/_helpers/nodes.rs | 29 ++- .../src/sensors/_helpers/transitions.rs | 56 ++++++ .../src/sensors/models.rs | 2 + .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/node_transitions.sql | 4 + .../tests/api/test_location_work.rs | 4 +- .../tests/api/warehouse_metadata/helpers.rs | 9 +- .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_runs.rs | 4 + .../tests/sensors/test_transitions.rs | 184 ++++++++++++++++++ .../tests/snapshots/helpers.rs | 1 + .../orchestration_declarations/types.py | 4 +- .../test_orchestration_snapshot.py | 28 ++- 19 files changed, 350 insertions(+), 20 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/check_transitions.json create mode 100644 crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs create mode 100644 crates/sqlbuild-orchestration/src/store/node_transitions.sql create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/check_transitions.json b/benchmarks/duckshop/edge_cases/orchestration/check_transitions.json new file mode 100644 index 0000000000..98838e4419 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/check_transitions.json @@ -0,0 +1,13 @@ +[ + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["failed", "failed"], "started": [true, true], "outcome": "failed"}, + {"nodes": ["success", "cancelled"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["success", "skipped"], "started": [true, false], "outcome": "partial"}, + {"nodes": ["success", "warn"], "started": [true, true], "outcome": "warn"}, + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["success", "success"], "started": [true, false], "outcome": "success"}, + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["success", "success"], "started": [true, true], "outcome": "success"}, + {"nodes": ["success", "success"], "started": [true, true], "outcome": "success"}, + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"} +] diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index dfa4e2427e..56cbc83f6f 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -497,7 +497,7 @@ export interface components { applied: boolean; }; /** @enum {string} */ - NodeEventKind: "success"; + NodeEventKind: "success" | "failed" | "recovered"; /** @enum {string} */ NodeOutcome: "success" | "warn" | "failed" | "skipped" | "cancelled"; /** @description Output of one stream that was not recorded because it exceeded `max_output_bytes`. */ diff --git a/crates/README.md b/crates/README.md index 28ea346d8b..9da43c9fec 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,23 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Individual node and check transitions + +`NodeEventKind.FAILED` matches entry into failure; repeated failures do not retrigger it. +`RECOVERED` matches an actual successful execution after a failure of that same node/check, +location and target, independently of its job or whole-run outcome. `SUCCESS` still matches +each genuine successful execution. A prior start in the same worker claim is required: +cached, blocked or unexecuted work cannot create a failure or claim a recovery. + +WARN is not failure. WARN, SKIPPED and CANCELLED do not clear an earlier actual failure; +cancellation remains in execution history but is not itself an active health problem. +Transitions are pure queries over immutable committed start/finish facts in storage order, +using the latest prior decisive execution, not a persisted mutable health flag. Cursor reset +recomputes the same transitions and existing run-key uniqueness prevents duplicate runs. +These consumers require individual committed node facts. Publishing Python checks into +worker snapshots and defining their frozen membership/retry behavior is a separate execution +boundary; the consumers do not infer check results from whole-build exit codes or diagnostic text. + ## Control-plane live updates `GET /api/locations/{location_id}/targets/{target}/changes` streams authenticated SSE diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs index 2e4243170e..b5b15de45e 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -20,8 +20,8 @@ pub(crate) async fn consume_condition( SensorCondition::PlatformEvent { kinds } => { consume_platform_events(transaction, evaluation, kinds).await } - SensorCondition::NodeEvent { selection, .. } => { - consume_node_events(transaction, evaluation, selection).await + SensorCondition::NodeEvent { selection, kinds } => { + consume_node_events(transaction, evaluation, selection, kinds).await } SensorCondition::RunEvent { job_key, diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs index 720f46154e..08edfd3e68 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -7,3 +7,4 @@ pub(crate) mod nodes; pub(crate) mod platform; pub(crate) mod requests; pub(crate) mod runs; +pub(crate) mod transitions; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs index c7f7a97b92..0f2b89f63a 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -1,4 +1,4 @@ -//! Successful rebuilds require earlier immutable start evidence from the same run claim. +//! Node transitions require earlier immutable start evidence from the same run claim. use std::collections::BTreeSet; @@ -6,7 +6,10 @@ use uuid::Uuid; use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; use crate::sensors::_helpers::requests::queue_sensor_run; -use crate::sensors::models::{NodeSensorEvent, SensorCursor, SensorEvaluationRequest}; +use crate::sensors::_helpers::transitions::matches_transition; +use crate::sensors::models::{ + NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, +}; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; @@ -16,6 +19,7 @@ pub(crate) async fn consume_node_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, selection: &BTreeSet, + kinds: &BTreeSet, ) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { let (mut transaction, rows) = query( transaction, @@ -37,20 +41,31 @@ pub(crate) async fn consume_node_events( return Err(ControlError::Invalid("node event identity mismatch".into())); } let RunChange::NodeFinished { - node_key, - outcome: NodeOutcome::Success, - .. + node_key, outcome, .. } = &request.change else { continue; }; - if !selection.contains(node_key) { + if !selection.contains(node_key) + || !matches!(outcome, NodeOutcome::Success | NodeOutcome::Failed) + { continue; } let started; (transaction, started) = prior_start(transaction, evaluation, &request, event.storage_id).await?; - if started { + if !started { + continue; + } + let matches; + (transaction, matches) = matches_transition( + transaction, + evaluation, + (node_key, *outcome, event.storage_id), + kinds, + ) + .await?; + if matches { let run_id; (transaction, run_id) = queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs new file mode 100644 index 0000000000..5c9becb4e4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs @@ -0,0 +1,56 @@ +//! Failure health is derived from decisive executed facts, never a mutable health flag. + +use std::collections::BTreeSet; + +use crate::runs::models::NodeOutcome; +use crate::sensors::models::{NodeEventKind, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn matches_transition( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + (node_key, outcome, storage_id): (&str, NodeOutcome, i64), + kinds: &BTreeSet, +) -> Result<(ControlTransaction, bool), ControlError> { + if outcome == NodeOutcome::Success && kinds.contains(&NodeEventKind::Success) { + return Ok((transaction, true)); + } + let kind = match outcome { + NodeOutcome::Success => NodeEventKind::Recovered, + NodeOutcome::Failed => NodeEventKind::Failed, + _ => return Ok((transaction, false)), + }; + if !kinds.contains(&kind) { + return Ok((transaction, false)); + } + let (transaction, rows) = query( + transaction, + "SELECT e.payload -> 'change' -> 'change' ->> 'outcome' AS outcome \ + FROM recall_events e JOIN sqb_runs r ON e.run_id=CAST(r.run_id AS TEXT) \ + WHERE e.producer='scheduler' AND e.event_type='run_node_finished' \ + AND e.invocation_id='scheduler/' || e.run_id AND e.storage_id<$1 \ + AND (e.payload -> 'change' -> 'change' ->> 'node_key')=$2 \ + AND (e.payload -> 'change' -> 'change' ->> 'outcome') IN ('success','failed') \ + AND r.location_id=$3 AND r.target=$4 AND r.kind<>'group' \ + AND EXISTS (SELECT 1 FROM recall_events s \ + WHERE s.invocation_id=e.invocation_id AND s.run_id=e.run_id \ + AND s.producer='scheduler' AND s.event_type='run_node_started' \ + AND s.storage_id 'change' ->> 'claim_id')=(e.payload -> 'change' ->> 'claim_id') \ + AND (s.payload -> 'change' -> 'change' ->> 'node_key')=$2) \ + ORDER BY e.storage_id DESC LIMIT 1", + vec![ + Parameter::Integer(storage_id), + Parameter::Text(node_key.into()), + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + ], + ) + .await?; + let failed = rows + .first() + .is_some_and(|row| row["outcome"] == NodeOutcome::Failed.as_str()); + Ok((transaction, (outcome == NodeOutcome::Success) == failed)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 38fbfced25..9c6a42d596 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -57,6 +57,8 @@ pub enum SensorCondition { #[serde(rename_all = "snake_case")] pub enum NodeEventKind { Success, + Failed, + Recovered, } #[derive( diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 8e208c08f5..b233c2fc4c 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -17,6 +17,7 @@ pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshness.sql"); pub(crate) const WAREHOUSE_FINGERPRINTS: &str = include_str!("../warehouse_fingerprints.sql"); pub(crate) const DEVELOPMENT_LOCATIONS: &str = include_str!("../development_locations.sql"); +pub(crate) const NODE_TRANSITIONS: &str = include_str!("../node_transitions.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 895f5e2304..4927f777f4 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, - render, verify_migrations, + NODE_TRANSITIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, + WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -40,6 +40,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_freshness", WAREHOUSE_FRESHNESS), ("warehouse_fingerprints", WAREHOUSE_FINGERPRINTS), ("development_locations", DEVELOPMENT_LOCATIONS), + ("node_transitions", NODE_TRANSITIONS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/node_transitions.sql b/crates/sqlbuild-orchestration/src/store/node_transitions.sql new file mode 100644 index 0000000000..f68711cc51 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/node_transitions.sql @@ -0,0 +1,4 @@ +CREATE INDEX sqb_node_decisive_history ON recall_events( + (payload -> 'change' -> 'change' ->> 'node_key'), storage_id DESC +) WHERE producer='scheduler' AND event_type='run_node_finished' +AND (payload -> 'change' -> 'change' ->> 'outcome') IN ('success','failed'); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 133fd8774c..d4bcb34632 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -49,12 +49,12 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable execute(store, "DROP TABLE sqb_development_locations").await; execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; execute(store, "DROP TABLE sqb_warehouse_freshness").await; - execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + execute(store, "DROP INDEX sqb_node_decisive_history; DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; migrate_store(store).await.expect("schema 2 upgrade"); execute(store, "DROP TABLE sqb_development_locations").await; execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; execute(store, "DROP TABLE sqb_warehouse_freshness").await; - execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + execute(store, "DROP INDEX sqb_node_decisive_history; DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; migrate_store(store).await.expect("schema 1 upgrade"); migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index f8951d639c..9c7936a7ac 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -23,17 +23,20 @@ pub(crate) async fn fingerprint_history( // Upgrade the immediate predecessor while published snapshots and freshness facts exist. let (transaction, _) = query( begin(store, TransactionMode::Write).await.expect("upgrade"), - "DROP TABLE sqb_development_locations", + "DROP INDEX sqb_node_decisive_history", vec![], ) .await - .expect("remove subsequent development migration"); + .expect("remove subsequent transition migration"); + let (transaction, _) = query(transaction, "DROP TABLE sqb_development_locations", vec![]) + .await + .expect("remove subsequent development migration"); let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_fingerprints", vec![]) .await .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 84e211d230..98c7d70d14 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -14,5 +14,7 @@ mod test_nodes; mod test_recovery; #[path = "test_runs.rs"] mod test_runs; +#[path = "test_transitions.rs"] +mod test_transitions; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 67f59043d0..947bcd1c60 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,6 +425,10 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_decisive_history", vec![]) + .await + .expect("remove subsequent transition migration"); let (transaction, _) = query(transaction, "DROP TABLE sqb_development_locations", vec![]) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs b/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs new file mode 100644 index 0000000000..227d29b409 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs @@ -0,0 +1,184 @@ +//! Independent check health survives non-decisive outcomes, other scopes and cursor replay. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_independent_check_failures_when_nondecisive_results_arrive_then_only_executed_success_recovers() + { + let test_cases = [ + SensorTestCase { + description: "SQLite check transitions", + backend: Backend::Sqlite, + expected_runs: 5, + }, + SensorTestCase { + description: "PostgreSQL check transitions", + backend: Backend::Postgres, + expected_runs: 5, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders check").kind = "check".into(); + input + .nodes + .get_mut("customers") + .expect("customers check") + .kind = "check".into(); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source checks", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "React to a transition", "selection": ["inventory"], "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "failed_checks": {"description": "Actual failures", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["failed"]}, "minimum_interval_seconds": 1}, + "recovered_checks": {"description": "Actual recoveries", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["recovered"]}, "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input).await.expect("publish"); + for scope in [(other, "development"), (location, "staging")] { + let mut isolated = input.clone(); + isolated.location_id = scope.0; + isolated.target = scope.1.into(); + publish_snapshot(store, &isolated) + .await + .expect("isolated snapshot"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/check_transitions.json"); + let transitions: Vec = serde_json::from_str( + &std::fs::read_to_string(fixture_path).expect("DuckShop fixture"), + ) + .expect("DuckShop independent check lifecycle"); + for transition in transitions { + finish_run( + store, + &lease, + (location, "development"), + "orders", + serde_json::from_value(transition["outcome"].clone()).expect("run outcome"), + serde_json::from_value(transition["nodes"].clone()).expect("check outcomes"), + serde_json::from_value(transition["started"].clone()).expect("actual starts"), + ) + .await; + // Another location/target cannot clear either check's actual failure. + for scope in [(other, "development"), (location, "staging")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + } + } + let history = read_recall_events(store, 0, 1000).await.expect("history"); + assert_eq!( + history + .iter() + .filter(|event| event["event_type"] == "run_node_finished" + && event["payload"]["change"]["change"]["outcome"] == "cancelled") + .count(), + 1 + ); + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + let mut queued = 0; + for trigger in &triggers { + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluate") + .expect("due"); + let count = tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &pending), + ) + .await + .expect("bounded transition page") + .expect("complete"); + assert_eq!( + count, + if trigger.name == "failed_checks" { + 3 + } else { + 2 + }, + "{}", + test_case.description + ); + queued += count; + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + } + assert_eq!(queued, test_case.expected_runs); + tokio::time::sleep(Duration::from_secs(1)).await; + for trigger in triggers { + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("deduplicated replay"), + if trigger.name == "failed_checks" { + 3 + } else { + 2 + } + ); + } + assert_eq!( + list_runs(store, (location, "development"), 1000) + .await + .expect("replayed runs") + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 91f6eb9314..a6f3b0dbe3 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -244,6 +244,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { connection .execute_batch( "DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ + DROP INDEX sqb_node_decisive_history; \ DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index 4635074185..71a09da2fe 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -27,6 +27,8 @@ class RunEventKind(StrEnum): class NodeEventKind(StrEnum): - """SUCCESS requires a genuine rebuild, not freshness-only completion.""" + """Actual execution drives success and transitions into or out of failure.""" SUCCESS = "success" + FAILED = "failed" + RECOVERED = "recovered" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 6f6cc7a182..bd973f07a7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -717,6 +717,30 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="failure and recovery selectors preserve their individual transition kinds", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace("NodeEventKind.SUCCESS", "NodeEventKind.FAILED, NodeEventKind.RECOVERED") + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["failed", "recovered"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", files={ @@ -955,13 +979,13 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_message="node_event needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( - description="node event kinds do not imply unapproved failure transitions", + description="unknown node event kinds cannot publish", files={ "orchestration/orders.py": Path( "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" ) .read_text() - .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['failed']") + .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['unknown_event']") }, expected_error=ValueError, expected_message="unknown variant", From 682b0437e820978720fe7776ebdf98001520c147 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:01:01 +0000 Subject: [PATCH 066/138] fix: restrict principal API tokens to service accounts --- .../orchestration/legacy_person_token.sql | 13 ++++++ crates/README.md | 8 ++++ .../src/credentials/_helpers/scopes.rs | 3 +- .../src/credentials/models.rs | 14 +++++-- .../tests/api/test_reads.rs | 11 +++-- .../tests/credentials/test_contract.rs | 42 +++++++++++++++++++ 6 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/legacy_person_token.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_person_token.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_person_token.sql new file mode 100644 index 0000000000..073f5242fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_person_token.sql @@ -0,0 +1,13 @@ +-- Synthetic predecessor rows exercise rejection without deleting credential history. +INSERT INTO sqb_principals (principal_id, name, kind, created_at) +VALUES ('00000000-0000-4000-8000-000000000001', 'orders-operator', 'person', '2026-01-01T00:00:00Z'); +INSERT INTO sqb_access_tokens + (token_id, secret_hash, principal_id, scope_json, created_at, expires_at) +VALUES ( + '00000000-0000-4000-8000-000000000002', + '17a07b942485f9d9e9a230b900005ceec1f6b4864574bd19783165c9c7ca1da3', + '00000000-0000-4000-8000-000000000001', + '{"kind":"principal","principal_id":"00000000-0000-4000-8000-000000000001"}', + '2026-01-01T00:00:00Z', + '2999-01-01T00:00:00Z' +); diff --git a/crates/README.md b/crates/README.md index 9da43c9fec..7f3820572c 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,14 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Service credentials + +Principal API tokens require an active service account at issuance and on every authenticated +request. Person sessions are a separate credential mechanism. Existing personal API-token rows +remain in storage for history but cannot authenticate; disabling a service account also +invalidates its tokens across replicas without a process-local cache. Location and run-claim +credentials retain their separate scopes and lifecycle checks. + ## Individual node and check transitions `NodeEventKind.FAILED` matches entry into failure; repeated failures do not retrigger it. diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs index 151288c4c4..f32eb48f08 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs @@ -23,7 +23,8 @@ pub(crate) async fn scope_is_active( ) -> Result<(ControlTransaction, bool), ControlError> { let (sql, parameters) = match scope { CredentialScope::Principal { principal_id } => ( - "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 \ + AND kind='service_account' AND disabled_at IS NULL", vec![Parameter::Id(principal_id)], ), CredentialScope::Location { location_id } => ( diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index f8f7d956f1..f32ea97224 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -16,9 +16,17 @@ pub enum PrincipalKind { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum CredentialScope { - Principal { principal_id: Uuid }, - Location { location_id: Uuid }, - Run { run_id: Uuid, claim_id: Uuid }, + /// API tokens belong to service accounts, never person sessions. + Principal { + principal_id: Uuid, + }, + Location { + location_id: Uuid, + }, + Run { + run_id: Uuid, + claim_id: Uuid, + }, } /// Server-derived request metadata, never copied from an action's JSON body. diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index 1a3b7a06b9..c1c6eaa049 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -140,10 +140,13 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso ) .await .expect("location token"); - let principal_id = - create_principal(store, &audit, ("orders operator", PrincipalKind::Person)) - .await - .expect("unprivileged person"); + let principal_id = create_principal( + store, + &audit, + ("orders automation", PrincipalKind::ServiceAccount), + ) + .await + .expect("unprivileged service account"); let principal = issue_token( store, &audit, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index d26aa25639..4e1e3252ac 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -17,6 +17,8 @@ use crate::credentials::tests::helpers::{location_state, stored_hashes}; use crate::credentials::tests::test_types::{CredentialTestCase, TransactionFault}; use crate::runs::tests::helpers::{queue, request, snapshot}; use crate::runs::tests::terminal_helpers::finish; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::TransactionMode; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; @@ -51,6 +53,46 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the request_id: Uuid::now_v7(), ip_address: Some("127.0.0.1".parse().expect("loopback")), }; + let predecessor = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/legacy_person_token.sql"); + let sql = std::fs::read_to_string(predecessor).expect("DuckShop predecessor token"); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("predecessor"); + for statement in sql.split(';').map(str::trim).filter(|sql| !sql.is_empty()) { + (transaction, _) = query(transaction, statement, vec![]) + .await + .expect("legacy person and hashed token"); + } + commit(transaction).await.expect("predecessor committed"); + let personal_scope = CredentialScope::Principal { + principal_id: Uuid::parse_str("00000000-0000-4000-8000-000000000001") + .expect("synthetic person identity"), + }; + assert!( + issue_token( + store, + &context, + personal_scope, + Utc::now() + Duration::hours(1) + ) + .await + .is_err(), + "people cannot mint API tokens" + ); + assert_eq!( + stored_hashes(&replica).await.len(), + 1, + "rejected issuance writes nothing" + ); + let legacy_bearer = format!("00000000-0000-4000-8000-000000000002.{}", "0".repeat(64)); + assert!( + authenticate_token(&replica, &legacy_bearer) + .await + .expect("predecessor personal credential") + .is_none(), + "pre-existing personal tokens are rejected on every replica" + ); let fault = TransactionFault { description: "security audit failure", table: "sqb_security_audit", From 310baf773cfb6ee4044a9e8f5ab5af01684eb10b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:44:03 +0000 Subject: [PATCH 067/138] test: align CLI credentials and catalog diagnostic expectations --- .../tests/native_commands/admin/test_admin.rs | 77 +++++++++++++++---- .../tests/native_commands/admin/test_types.rs | 2 +- .../compiler/failure/target-unknown-key.json | 2 +- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index f2ab692398..cb22985e20 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -161,23 +161,72 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope } } +#[test] +fn given_native_person_administration_when_issuing_an_api_credential_then_it_is_rejected_without_issuance_auditing() + { + let test_cases = [PrincipalAdminTestCase { + description: "a person cannot receive an API credential", + kind: "person", + expected_name: "orders operator / café", + expected_actions: &["principal.created"], + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + api_ready(&mut control).await; + let principal = administer( + &control, + &[ + "create-principal", + test_case.expected_name, + "--kind", + test_case.kind, + ], + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let output = invoke_admin( + &control, + &[ + "issue-principal-token", + principal["principal_id"] + .as_str() + .expect(test_case.description), + "--expires-at", + &expiry, + ], + ); + assert!(!output.status.success(), "{}", test_case.description); + assert!(output.stdout.is_empty(), "{}", test_case.description); + assert!( + String::from_utf8_lossy(&output.stderr).contains("token scope is not active"), + "{}", + test_case.description + ); + assert_eq!( + audit_actions(&control, "no-issued-person-credential"), + test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + #[test] fn given_native_principal_administration_when_issuing_credentials_then_identity_is_audited_without_granting_location_permissions() { - let test_cases = [ - PrincipalAdminTestCase { - description: "a person credential has no implicit location authority", - kind: "person", - expected_name: "orders operator / café", - expected_actions: ["principal.created", "token.issued", "token.revoked"], - }, - PrincipalAdminTestCase { - description: "a service account credential has no implicit location authority", - kind: "service_account", - expected_name: "orders automation", - expected_actions: ["principal.created", "token.issued", "token.revoked"], - }, - ]; + let test_cases = [PrincipalAdminTestCase { + description: "a service account credential has no implicit location authority", + kind: "service_account", + expected_name: "orders automation", + expected_actions: &["principal.created", "token.issued", "token.revoked"], + }]; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs index 3adc9b5e24..8f0c87914a 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -9,7 +9,7 @@ pub(crate) struct PrincipalAdminTestCase { pub(crate) description: &'static str, pub(crate) kind: &'static str, pub(crate) expected_name: &'static str, - pub(crate) expected_actions: [&'static str; 3], + pub(crate) expected_actions: &'static [&'static str], } pub(crate) struct TriggerAdminTestCase { diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 036d4c22ef..457bf232d0 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] From 4448c718a8f12e0d2358113520a546ad7ae38d1e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:16:19 +0000 Subject: [PATCH 068/138] feat: add native password and browser session foundation Add opt-in Argon2id login, hashed secure sessions, independent CSRF, shared login reservations and transactional security audits. Revalidate password versions and person eligibility across replicas without granting human sessions implicit resource authority. Cover SQLite and PostgreSQL lifecycle, fault and HTTP boundary contracts and regenerate the native TypeScript schema. --- Cargo.lock | 39 ++ Cargo.toml | 1 + .../orchestration/password_credentials.json | 14 + clients/typescript/src/schema.d.ts | 215 ++++++++++ clients/typescript/tests/types.ts | 6 + .../src/native_commands/_helpers/execution.rs | 7 +- .../src/native_commands/main/_run_control.rs | 3 +- crates/sqlbuild-orchestration/Cargo.toml | 1 + .../bearer.rs} | 2 +- .../api/_helpers/authentication/configure.rs | 32 ++ .../src/api/_helpers/authentication/mod.rs | 4 + .../authentication/password_sessions.rs | 203 ++++++++++ .../authentication/password_transport.rs | 149 +++++++ .../src/api/_helpers/mod.rs | 10 +- .../api/_helpers/{ => resources}/changes.rs | 2 +- .../api/_helpers/{ => resources}/client.rs | 2 +- .../{ => resources}/location_health.rs | 2 +- .../_helpers/{ => resources}/location_work.rs | 2 +- .../src/api/_helpers/resources/mod.rs | 10 + .../src/api/_helpers/resources/router.rs | 86 ++++ .../api/_helpers/{ => resources}/run_reads.rs | 2 +- .../src/api/_helpers/{ => resources}/runs.rs | 2 +- .../api/_helpers/{ => resources}/snapshots.rs | 2 +- .../_helpers/{ => resources}/trigger_reads.rs | 2 +- .../{ => resources}/warehouse_metadata.rs | 2 +- .../src/api/constants.rs | 7 + .../src/api/main/connect_client.rs | 2 +- .../src/api/main/openapi_document.rs | 10 +- .../src/api/main/serve_api.rs | 107 +---- .../sqlbuild-orchestration/src/api/models.rs | 68 +++- .../src/credentials/_helpers/mod.rs | 3 + .../credentials/_helpers/password_sessions.rs | 69 ++++ .../src/credentials/_helpers/passwords.rs | 55 +++ .../src/credentials/_helpers/secrets.rs | 43 ++ .../src/credentials/constants.rs | 12 + .../credentials/main/authenticate_session.rs | 36 ++ .../credentials/main/authenticate_token.rs | 19 +- .../src/credentials/main/issue_token.rs | 13 +- .../src/credentials/main/login_password.rs | 62 +++ .../src/credentials/main/mod.rs | 7 + .../main/record_authentication_denial.rs | 33 ++ .../main/reserve_password_attempt.rs | 72 ++++ .../src/credentials/main/revoke_session.rs | 51 +++ .../src/credentials/main/set_password.rs | 96 +++++ .../src/credentials/main/verify_password.rs | 51 +++ .../src/credentials/models.rs | 64 +++ .../src/store/_helpers/migration.rs | 2 + .../src/store/main/migrate_store.rs | 6 +- .../src/store/password_credentials.sql | 6 + .../src/store/password_sessions.sql | 20 + .../tests/api/helpers.rs | 10 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_development.rs | 19 +- .../tests/api/test_location_work.rs | 24 +- .../tests/api/test_passwords.rs | 349 ++++++++++++++++ .../tests/api/test_types.rs | 6 + .../tests/api/warehouse_metadata/helpers.rs | 21 +- .../credentials/passwords/test_contract.rs | 286 +++++++++++++ .../tests/credentials/passwords/test_types.rs | 7 + .../credentials/sessions/test_contract.rs | 377 ++++++++++++++++++ .../tests/credentials/sessions/test_types.rs | 8 + .../tests/credentials/test_contracts.rs | 4 + .../tests/sensors/test_runs.rs | 11 +- .../tests/snapshots/helpers.rs | 3 +- fensu.toml | 10 +- 65 files changed, 2668 insertions(+), 183 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/password_credentials.json rename crates/sqlbuild-orchestration/src/api/_helpers/{authentication.rs => authentication/bearer.rs} (98%) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/changes.rs (98%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/client.rs (92%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/location_health.rs (96%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/location_work.rs (97%) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/run_reads.rs (98%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/runs.rs (98%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/snapshots.rs (98%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/trigger_reads.rs (97%) rename crates/sqlbuild-orchestration/src/api/_helpers/{ => resources}/warehouse_metadata.rs (97%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/login_password.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/set_password.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/verify_password.rs create mode 100644 crates/sqlbuild-orchestration/src/store/password_credentials.sql create mode 100644 crates/sqlbuild-orchestration/src/store/password_sessions.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_passwords.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs diff --git a/Cargo.lock b/Cargo.lock index ef08978855..839ea3ccd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,18 @@ dependencies = [ "object", ] +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.2.17", + "password-hash", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -166,6 +178,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.11.1" @@ -191,6 +209,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1539,6 +1566,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2585,6 +2623,7 @@ dependencies = [ name = "sqlbuild-orchestration" version = "0.186.2" dependencies = [ + "argon2", "axum", "chrono", "chrono-tz", diff --git a/Cargo.toml b/Cargo.toml index c5f2325b60..d6680590b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ publish = false rust-version = "1.88" [workspace.dependencies] +argon2 = "0.5.3" axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } fensu-policy = "=0.22.0" futures-util = { version = "0.3", default-features = false, features = ["std"] } diff --git a/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json b/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json new file mode 100644 index 0000000000..0f690df069 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json @@ -0,0 +1,14 @@ +{ + "password": "synthetic-orders-password", + "replacement": "synthetic-customers-password", + "wrong": "synthetic-inventory-password", + "too_short": "订单客户库存", + "minimum_unicode": "订单客户库存发货支持产品", + "username": "orders-reader", + "origin": "https://control.example.com", + "maximum_password_bytes": 1024, + "http_password_character": "\u0000", + "http_replacement_character": "\u0001", + "invalid_origins": ["https://other.example.com", "null"], + "invalid_csrf": ["", "not-a-session-secret"] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 56cbc83f6f..683f5af421 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1,5 +1,69 @@ // Generated from sqb server --openapi. Run npm run generate; do not edit. export interface paths { + "/api/auth/logout": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["logoutSession"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/password": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["changePassword"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/password/login": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["loginPassword"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/session": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["currentSession"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/health": { parameters: { query?: never; @@ -327,6 +391,11 @@ export interface components { ApiErrorBody: { error: string; }; + AuthenticationDenial: { + /** Format: uuid */ + decision_id: string; + error: string; + }; CatchUp: "latest" | { /** @description Process the oldest bounded batch; later due times remain due for the next transaction. */ all: { @@ -548,6 +617,18 @@ export interface components { OutputStream: "stdout" | "stderr"; /** @enum {string} */ Overlap: "coalesce" | "queue" | "allow"; + PasswordChangeRequest: { + current_password: string; + new_password: string; + }; + PasswordLoginRequest: { + password: string; + username: string; + }; + PasswordSessionResponse: { + csrf_token: string; + identity: components["schemas"]["SessionIdentity"]; + }; /** @enum {string} */ PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; /** @description One invocation's records; the server owns its claim namespace, start time and version. */ @@ -802,6 +883,14 @@ export interface components { }; /** @enum {string} */ SensorStatus: "running" | "stopped"; + SessionIdentity: { + /** Format: date-time */ + expires_at: string; + /** Format: uuid */ + principal_id: string; + /** Format: uuid */ + session_id: string; + }; /** @description Node-level changes do not duplicate unchanged definitions or source text. */ SnapshotChange: { after_hash?: string | null; @@ -988,6 +1077,132 @@ export interface components { } export type $defs = Record; export interface operations { + logoutSession: { + parameters: { + query?: never; + header: { + /** @description Session CSRF token */ + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + changePassword: { + parameters: { + query?: never; + header: { + /** @description Session CSRF token */ + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PasswordChangeRequest"]; + }; + }; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + loginPassword: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PasswordLoginRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PasswordSessionResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + currentSession: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SessionIdentity"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; read_location_health: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 5ad1a573b7..55ff245107 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -9,3 +9,9 @@ void client.GET("/api/locations/{location_id}/health", {}); // @ts-expect-error Only the native discriminated health variants are accepted. const invalid: components["schemas"]["LocationHealth"] = { kind: "unknown" }; void invalid; +// @ts-expect-error Session mutations require the CSRF header. +void client.POST("/api/auth/logout", {}); +// @ts-expect-error Changing a password also requires the current password. +void client.POST("/api/auth/password", { params: { header: { "x-sqb-csrf": "synthetic-csrf" } }, body: { new_password: "synthetic-password" } }); +// @ts-expect-error Session responses never expose a browser-readable bearer. +void ({} as components["schemas"]["PasswordSessionResponse"]).bearer; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 6d982826ca..939cddd409 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -9,7 +9,7 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::api::models::DevelopmentScope; +use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope}; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; @@ -86,7 +86,10 @@ pub(crate) async fn run_development_roles( tasks.spawn(serve_api( store.clone(), listener, - Some(scope), + ApiAuthentication { + development_scope: Some(scope), + ..ApiAuthentication::default() + }, development_stopped(shutdown.clone()), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index cc58bab029..c557ba3bce 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,6 +1,7 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::api::models::ApiAuthentication; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -45,7 +46,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - None, + ApiAuthentication::default(), stopped(shutdown.clone()), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 542bbd196b..2d7004113c 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +argon2.workspace = true axum.workspace = true chrono.workspace = true chrono-tz.workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs index 1bee960538..317bb68c88 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs @@ -1,4 +1,4 @@ -//! Authenticate before body buffering; scope checks remain in each owning endpoint. +//! Authenticate machine bearers before buffering; scope checks stay in each owning endpoint. use std::net::SocketAddr; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs new file mode 100644 index 0000000000..837ad3412c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs @@ -0,0 +1,32 @@ +//! Validate authentication modes before migrations or HTTP startup. + +use std::net::SocketAddr; + +use axum::Router; + +use crate::api::_helpers::authentication::password_sessions::router; +use crate::api::models::{ApiAuthentication, DevelopmentScope}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) fn configure( + store: &ControlStore, + authentication: ApiAuthentication, + address: SocketAddr, +) -> Result<(Option, Router), ControlError> { + let ApiAuthentication { + development_scope, + password_origin, + } = authentication; + if development_scope.is_some() && password_origin.is_some() { + return Err(ControlError::Invalid( + "disabled and password authentication cannot be combined".into(), + )); + } + if development_scope.is_some() && !address.ip().is_loopback() { + return Err(ControlError::Invalid( + "disabled authentication requires a loopback listener".into(), + )); + } + Ok((development_scope, router(store, password_origin)?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs new file mode 100644 index 0000000000..6684241c2a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod bearer; +pub(crate) mod configure; +pub(crate) mod password_sessions; +pub(crate) mod password_transport; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs new file mode 100644 index 0000000000..d9db92f253 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -0,0 +1,203 @@ +//! Password endpoints return an HttpOnly cookie, never a browser-readable session bearer. + +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Extension, Json, Router, middleware}; + +use crate::api::_helpers::authentication::password_transport::{authenticate, deny, guard}; +use crate::api::constants::{ + HTTPS_SCHEME, PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE, ROOT_PATH, +}; +use crate::api::errors::ApiError; +use crate::api::models::{ + AuthenticationDenial, PasswordChangeRequest, PasswordLoginRequest, PasswordOrigin, + PasswordSessionResponse, +}; +use crate::credentials::constants::{LOGIN_WINDOW_SECONDS, PASSWORD_SESSION_SECONDS}; +use crate::credentials::main::{ + login_password::login_password, reserve_password_attempt::reserve_password_attempt, + revoke_session::revoke_session, set_password::set_password, verify_password::verify_password, +}; +use crate::credentials::models::{ + AuditContext, PasswordAttempt, PasswordAuthority, PasswordLogin, SessionIdentity, +}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) fn router( + store: &ControlStore, + origin: Option, +) -> Result, ControlError> { + let Some(origin) = origin else { + return Ok(Router::new()); + }; + let url = reqwest::Url::parse(&origin).map_err(|_| { + ControlError::Invalid("password authentication requires an HTTPS origin".into()) + })?; + if url.scheme() != HTTPS_SCHEME + || url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != ROOT_PATH + || url.query().is_some() + || url.fragment().is_some() + { + return Err(ControlError::Invalid("password authentication requires an HTTPS origin without credentials, path, query or fragment".into())); + } + let protected = Router::new() + .route("/api/auth/session", get(current)) + .route("/api/auth/logout", post(logout)) + .route("/api/auth/password", post(change_password)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); + Ok(protected + .route("/api/auth/password/login", post(login)) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) + .layer(middleware::from_fn_with_state(store.clone(), guard)) + .layer(Extension(PasswordOrigin( + url.origin().ascii_serialization(), + )))) +} + +#[utoipa::path(post, path = "/api/auth/password/login", operation_id = "loginPassword", + request_body = PasswordLoginRequest, + responses((status = 200, body = PasswordSessionResponse), (status = 401, body = AuthenticationDenial), + (status = 429, body = AuthenticationDenial)))] +pub(crate) async fn login( + State(store): State, + Extension(context): Extension, + Json(request): Json, +) -> Result { + match login_password(&store, &context, (&request.username, &request.password)).await? { + PasswordLogin::Authenticated(session) => { + let mut response = Json(PasswordSessionResponse { + identity: session.identity, + csrf_token: session.csrf.clone(), + }) + .into_response(); + set_cookie( + &mut response, + (PASSWORD_COOKIE, &session.bearer), + PASSWORD_SESSION_SECONDS, + true, + )?; + set_cookie( + &mut response, + (PASSWORD_CSRF_COOKIE, &session.csrf), + PASSWORD_SESSION_SECONDS, + false, + )?; + Ok(response) + } + PasswordLogin::Denied { + decision_id, + throttled, + } => { + let mut response = ( + if throttled { + StatusCode::TOO_MANY_REQUESTS + } else { + StatusCode::UNAUTHORIZED + }, + Json(AuthenticationDenial { + error: "authentication rejected".into(), + decision_id, + }), + ) + .into_response(); + if throttled { + response.headers_mut().insert( + header::RETRY_AFTER, + HeaderValue::from_str(&LOGIN_WINDOW_SECONDS.to_string()).map_err(|_| { + ControlError::Invalid("invalid login retry interval".into()) + })?, + ); + } + Ok(response) + } + } +} + +#[utoipa::path(get, path = "/api/auth/session", operation_id = "currentSession", + responses((status = 200, body = SessionIdentity), (status = 401, body = AuthenticationDenial)), + security(("session" = [])))] +pub(crate) async fn current( + Extension(identity): Extension, +) -> Json { + Json(identity) +} + +#[utoipa::path(post, path = "/api/auth/logout", operation_id = "logoutSession", + params(("x-sqb-csrf" = String, Header, description = "Session CSRF token")), + responses((status = 204), (status = 401, body = AuthenticationDenial)), + security(("session" = [])))] +pub(crate) async fn logout( + State(store): State, + Extension(context): Extension, + Extension(identity): Extension, +) -> Result { + revoke_session(&store, &context, &identity).await?; + clear_cookies() +} + +#[utoipa::path(post, path = "/api/auth/password", operation_id = "changePassword", + params(("x-sqb-csrf" = String, Header, description = "Session CSRF token")), + request_body = PasswordChangeRequest, + responses((status = 204), (status = 401, body = AuthenticationDenial)), + security(("session" = [])))] +pub(crate) async fn change_password( + State(store): State, + Extension(context): Extension, + Extension(identity): Extension, + Json(request): Json, +) -> Result { + let (_, allowed) = reserve_password_attempt( + &store, + &context, + PasswordAttempt::Reauthenticate(identity.principal_id), + ) + .await?; + if !allowed { + return deny(&store, &context, "reauthentication_throttled").await; + } + let Some(proof) = + verify_password(&store, identity.principal_id, &request.current_password).await? + else { + return deny(&store, &context, "invalid_current_password").await; + }; + set_password( + &store, + &context, + PasswordAuthority::Reauthenticated { + proof, + session_id: identity.session_id, + }, + &request.new_password, + ) + .await?; + clear_cookies() +} + +fn clear_cookies() -> Result { + let mut response = StatusCode::NO_CONTENT.into_response(); + set_cookie(&mut response, (PASSWORD_COOKIE, ""), 0, true)?; + set_cookie(&mut response, (PASSWORD_CSRF_COOKIE, ""), 0, false)?; + Ok(response) +} + +fn set_cookie( + response: &mut Response, + (name, value): (&str, &str), + max_age: i64, + http_only: bool, +) -> Result<(), ApiError> { + let attributes = if http_only { "; HttpOnly" } else { "" }; + let mut cookie = HeaderValue::from_str(&format!( + "{name}={value}; Path=/; Secure; SameSite=Strict; Max-Age={max_age}{attributes}" + )) + .map_err(|_| ControlError::Invalid("could not encode a session cookie".into()))?; + cookie.set_sensitive(true); + response.headers_mut().append(header::SET_COOKIE, cookie); + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs new file mode 100644 index 0000000000..7f105fdec1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -0,0 +1,149 @@ +//! Browser credentials use host-only cookies; peer identity never comes from forwarded headers. + +use std::net::SocketAddr; + +use axum::extract::{ConnectInfo, Request, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use axum::{Extension, Json}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::constants::{PASSWORD_COOKIE, PASSWORD_CSRF_HEADER}; +use crate::api::errors::ApiError; +use crate::api::models::{AuthenticationDenial, PasswordOrigin}; +use crate::credentials::main::authenticate_session::authenticate_session; +use crate::credentials::main::record_authentication_denial::record_authentication_denial; +use crate::credentials::models::{AuditContext, SessionAccess}; +use crate::store::models::ControlStore; + +pub(crate) async fn guard( + State(store): State, + Extension(origin): Extension, + mut request: Request, + next: Next, +) -> Result { + let peer = request + .extensions() + .get::>() + .ok_or_else(|| ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "trusted request peer is unavailable".into(), + })? + .0; + let context = AuditContext { + actor: "anonymous".into(), + granting_role: "authentication".into(), + request_id: Uuid::now_v7(), + ip_address: Some(peer.ip().to_canonical()), + }; + let valid_host = single_header(request.headers(), header::HOST.as_str()) + == origin.0.strip_prefix("https://"); + let valid_origin = match single_header(request.headers(), header::ORIGIN.as_str()) { + Some(value) => value == origin.0, + None => request.method().is_safe() && !request.headers().contains_key(header::ORIGIN), + }; + let mut response = if !valid_host || !valid_origin { + deny(&store, &context, "invalid_origin").await? + } else { + request.extensions_mut().insert(context); + next.run(request).await + }; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + response.headers_mut().insert( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + Ok(response) +} + +pub(crate) async fn authenticate( + State(store): State, + mut request: Request, + next: Next, +) -> Result { + let context = request + .extensions() + .get::() + .cloned() + .ok_or_else(|| ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "trusted authentication context is unavailable".into(), + })?; + let access = if request.method().is_safe() { + SessionAccess::Read + } else { + SessionAccess::Write { + csrf: single_header(request.headers(), PASSWORD_CSRF_HEADER).unwrap_or_default(), + } + }; + let identity = authenticate_session( + &store, + session_cookie(request.headers()).unwrap_or_default(), + access, + ) + .await?; + let Some(identity) = identity else { + return deny(&store, &context, "invalid_session_or_csrf").await; + }; + let mut context = context; + context.actor = format!("principal:{}", identity.principal_id); + context.granting_role = "password-session".into(); + request.extensions_mut().insert(context); + request.extensions_mut().insert(identity); + Ok(next.run(request).await) +} + +pub(crate) fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + let mut values = headers.get_all(name).iter(); + let Ok(value) = values.next()?.to_str() else { + return None; + }; + values.next().is_none().then_some(value) +} + +fn session_cookie(headers: &HeaderMap) -> Option<&str> { + let mut session = None; + for header in headers.get_all(header::COOKIE) { + let Ok(value) = header.to_str() else { + return None; + }; + for cookie in value.split(';') { + let Some((name, value)) = cookie.trim().split_once('=') else { + continue; + }; + if name == PASSWORD_COOKIE { + if session.is_some() { + return None; + } + session = Some(value); + } + } + } + session +} + +pub(crate) async fn deny( + store: &ControlStore, + context: &AuditContext, + reason: &str, +) -> Result { + let decision_id = record_authentication_denial( + store, + context, + (context.request_id, reason), + json!({"kind": "browser_authentication"}), + ) + .await?; + Ok(( + StatusCode::UNAUTHORIZED, + Json(AuthenticationDenial { + error: "authentication rejected".into(), + decision_id, + }), + ) + .into_response()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs index 0f6623e274..528c844b6e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -1,10 +1,2 @@ pub(crate) mod authentication; -pub(crate) mod changes; -pub(crate) mod client; -pub(crate) mod location_health; -pub(crate) mod location_work; -pub(crate) mod run_reads; -pub(crate) mod runs; -pub(crate) mod snapshots; -pub(crate) mod trigger_reads; -pub(crate) mod warehouse_metadata; +pub(crate) mod resources; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/api/_helpers/changes.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs index 98a327eedd..3d5ffb7424 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs @@ -12,7 +12,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use tokio::sync::broadcast; use uuid::Uuid; -use crate::api::_helpers::authentication::{bearer, require_location}; +use crate::api::_helpers::authentication::bearer::{bearer, require_location}; use crate::api::errors::ApiError; use crate::api::models::{ChangeFeed, ChangeNotice, ChangeStream}; use crate::credentials::main::authenticate_token::authenticate_token; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/client.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/client.rs similarity index 92% rename from crates/sqlbuild-orchestration/src/api/_helpers/client.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/client.rs index 98532b9054..e2a86e2291 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/client.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/client.rs @@ -1,4 +1,4 @@ -//! Each client runtime owns its pool; credentials and server response bodies stay out of diagnostics. +//! Outbound clients own their pools; credentials and response bodies stay out of diagnostics. use std::time::Duration; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs index 65baf9a1a0..24a318b3f9 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs @@ -4,7 +4,7 @@ use axum::extract::{Path, State}; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::bearer::require_location; use crate::api::errors::ApiError; use crate::api::models::MutationResponse; use crate::credentials::models::TokenIdentity; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs index 6e48859e78..f6bfb530c3 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs @@ -4,7 +4,7 @@ use axum::extract::{Path, State}; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::bearer::require_location; use crate::api::errors::ApiError; use crate::api::models::MutationResponse; use crate::credentials::models::TokenIdentity; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs new file mode 100644 index 0000000000..71e23cbbb7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod changes; +pub(crate) mod client; +pub(crate) mod location_health; +pub(crate) mod location_work; +pub(crate) mod router; +pub(crate) mod run_reads; +pub(crate) mod runs; +pub(crate) mod snapshots; +pub(crate) mod trigger_reads; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs new file mode 100644 index 0000000000..757556ce89 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -0,0 +1,86 @@ +//! Machine resource routes share bearer authentication and bounded request bodies. + +use axum::extract::DefaultBodyLimit; +use axum::routing::{get, post}; +use axum::{Extension, Router, middleware}; + +use crate::api::_helpers::authentication::bearer::authenticate; +use crate::api::_helpers::resources::{ + changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, + warehouse_metadata, +}; +use crate::api::constants::BODY_BYTES; +use crate::api::models::ChangeFeed; +use crate::store::models::ControlStore; + +pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router { + Router::new() + .route( + "/api/locations/{location_id}/targets/{target}/changes", + get(changes::subscribe), + ) + .route( + "/api/locations/{location_id}/work/poll", + post(location_work::poll), + ) + .route( + "/api/locations/{location_id}/work/{request_id}", + get(location_work::result).post(location_work::complete), + ) + .route( + "/api/locations/{location_id}/health", + get(location_health::recent).post(location_health::report), + ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + post(warehouse_metadata::report), + ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/fingerprints", + post(warehouse_metadata::report_fingerprints), + ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots", + get(snapshots::current).post(snapshots::publish), + ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + get(snapshots::metadata), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs", + get(run_reads::recent), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + get(run_reads::nodes), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + get(run_reads::invocations), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + get(run_reads::recall), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers", + get(trigger_reads::current), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + get(trigger_reads::events), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + get(trigger_reads::ticks), + ) + .route("/api/runs/{run_id}/execution", get(runs::execution)) + .route("/api/runs/{run_id}/events", post(runs::report)) + .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .route("/api/runs/{run_id}/output", post(runs::output)) + .route("/api/runs/{run_id}/recall", post(runs::recall)) + .layer(DefaultBodyLimit::max(BODY_BYTES)) + .layer(Extension(feed)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs index 44c484838a..1cfc321bbd 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs @@ -5,7 +5,7 @@ use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::{require_location, require_resource}; +use crate::api::_helpers::authentication::bearer::{require_location, require_resource}; use crate::api::errors::ApiError; use crate::api::models::{ReadQuery, ReadResource, RunRecallPath, RunRecallQuery}; use crate::credentials::models::TokenIdentity; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/api/_helpers/runs.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs index 636ccc0f9f..14b109e765 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs @@ -1,4 +1,4 @@ -//! Worker reports are bound to the authenticated run and claim, never a caller-supplied actor. +//! Worker resource reports bind to the authenticated run and claim, never a caller-supplied actor. use axum::extract::{Path, State}; use axum::http::StatusCode; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs index 7c7e452aee..7e54518ab7 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs @@ -5,7 +5,7 @@ use axum::http::StatusCode; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::bearer::require_location; use crate::api::errors::ApiError; use crate::api::models::DevelopmentScope; use crate::credentials::models::TokenIdentity; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs index dcced622c8..a3bf1e5327 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs @@ -4,7 +4,7 @@ use axum::extract::{Path, Query, State}; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::{require_location, require_resource}; +use crate::api::_helpers::authentication::bearer::{require_location, require_resource}; use crate::api::errors::ApiError; use crate::api::models::{ReadQuery, ReadResource}; use crate::credentials::models::TokenIdentity; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs rename to crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs index 4645faec67..fa0606bb77 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs @@ -4,7 +4,7 @@ use axum::extract::{Path, State}; use axum::{Extension, Json}; use uuid::Uuid; -use crate::api::_helpers::authentication::require_location; +use crate::api::_helpers::authentication::bearer::require_location; use crate::api::errors::ApiError; use crate::api::models::MutationResponse; use crate::credentials::models::TokenIdentity; diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index d3d0c93f8d..7bb3346324 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -1,3 +1,10 @@ //! Shared HTTP transport bounds for the server and native clients. pub(crate) const BODY_BYTES: usize = 64 * 1024 * 1024; +/// Two 1024-byte passwords can each expand sixfold through JSON escaping. +pub(crate) const PASSWORD_BODY_BYTES: usize = 16 * 1024; +pub(crate) const PASSWORD_COOKIE: &str = "__Host-sqb-session"; +pub(crate) const PASSWORD_CSRF_COOKIE: &str = "__Host-sqb-csrf"; +pub(crate) const PASSWORD_CSRF_HEADER: &str = "x-sqb-csrf"; +pub(crate) const HTTPS_SCHEME: &str = "https"; +pub(crate) const ROOT_PATH: &str = "/"; diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs index c0b5258123..3c717eb4f6 100644 --- a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -2,7 +2,7 @@ use reqwest::header::HeaderValue; -use crate::api::_helpers::client::http_client; +use crate::api::_helpers::resources::client::http_client; use crate::api::errors::ClientError; use crate::api::models::ControlClient; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs index 597993eda8..707449538e 100644 --- a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -1,8 +1,9 @@ //! Generate the HTTP contract from the request/response types used by the actual endpoints. use utoipa::OpenApi; -use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; +use utoipa::openapi::security::{ApiKey, ApiKeyValue, HttpAuthScheme, HttpBuilder, SecurityScheme}; +use crate::api::constants::PASSWORD_COOKIE; use crate::api::models::ControlApi; #[must_use] @@ -16,4 +17,11 @@ pub fn openapi_document() -> utoipa::openapi::OpenApi { SecurityScheme::Http(HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build()), ); document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "session", + SecurityScheme::ApiKey(ApiKey::Cookie(ApiKeyValue::new(PASSWORD_COOKIE))), + ); + document } diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 319dad03bd..a1b98c1af5 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -1,17 +1,12 @@ //! One native server owns migrations and all storage writes; locations and workers speak HTTP. -use axum::extract::DefaultBodyLimit; -use axum::routing::{get, post}; +use axum::routing::get; use axum::{Extension, Json, Router, middleware}; -use crate::api::_helpers::authentication::{authenticate, local_request}; -use crate::api::_helpers::{ - changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, - warehouse_metadata, -}; -use crate::api::constants::BODY_BYTES; +use crate::api::_helpers::authentication::{bearer::local_request, configure::configure}; +use crate::api::_helpers::resources::{changes, router::router}; use crate::api::main::openapi_document::openapi_document; -use crate::api::models::{ChangeFeed, DevelopmentScope}; +use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope}; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; @@ -20,11 +15,11 @@ use crate::store::models::ControlStore; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, - development_scope: Option, + authentication: ApiAuthentication, shutdown: impl Future + Send + 'static, ) -> Result<(), ControlError> { let address = listener.local_addr()?; - validate_development_listener(address, &development_scope)?; + let (development_scope, password_routes) = configure(&store, authentication, address)?; migrate_store(&store).await?; let (sender, listener_changes) = subscribe_changes(&store).await?; let (stop, stopped) = tokio::sync::watch::channel(false); @@ -32,83 +27,19 @@ pub async fn serve_api( sender: sender.clone(), stopped, }; - let protected = Router::new() - .route( - "/api/locations/{location_id}/targets/{target}/changes", - get(changes::subscribe), - ) - .route( - "/api/locations/{location_id}/work/poll", - post(location_work::poll), - ) - .route( - "/api/locations/{location_id}/work/{request_id}", - get(location_work::result).post(location_work::complete), - ) - .route( - "/api/locations/{location_id}/health", - get(location_health::recent).post(location_health::report), - ) - .route( - "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", - post(warehouse_metadata::report), - ) - .route( - "/api/locations/{location_id}/targets/{target}/metadata/fingerprints", - post(warehouse_metadata::report_fingerprints), - ) - .route( - "/api/locations/{location_id}/targets/{target}/snapshots", - get(snapshots::current).post(snapshots::publish), - ) - .route( - "/api/locations/{location_id}/targets/{target}/snapshots/metadata", - get(snapshots::metadata), - ) - .route( - "/api/locations/{location_id}/targets/{target}/runs", - get(run_reads::recent), - ) - .route( - "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", - get(run_reads::nodes), - ) - .route( - "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", - get(run_reads::invocations), - ) - .route( - "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", - get(run_reads::recall), - ) - .route( - "/api/locations/{location_id}/targets/{target}/triggers", - get(trigger_reads::current), - ) - .route( - "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", - get(trigger_reads::events), - ) - .route( - "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", - get(trigger_reads::ticks), - ) - .route("/api/runs/{run_id}/execution", get(runs::execution)) - .route("/api/runs/{run_id}/events", post(runs::report)) - .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) - .route("/api/runs/{run_id}/output", post(runs::output)) - .route("/api/runs/{run_id}/recall", post(runs::recall)) - .layer(DefaultBodyLimit::max(BODY_BYTES)) - .layer(Extension(feed)) - .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); - let application = protected + let application = router(&store, feed) + .merge(password_routes) .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), ) .with_state(store.clone()); let application = development_application(application, development_scope, address); - let server = axum::serve(listener, application).with_graceful_shutdown(async move { + let server = axum::serve( + listener, + application.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { shutdown.await; stop.send_replace(true); }); @@ -118,18 +49,6 @@ pub async fn serve_api( } } -fn validate_development_listener( - address: std::net::SocketAddr, - scope: &Option, -) -> Result<(), ControlError> { - if scope.is_some() && !address.ip().is_loopback() { - return Err(ControlError::Invalid( - "disabled authentication requires a loopback listener".into(), - )); - } - Ok(()) -} - fn development_application( application: Router, scope: Option, diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 3b53aceaf0..c03eace64a 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,13 +2,18 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::{ +use crate::api::_helpers::authentication::password_sessions; +use crate::api::_helpers::resources::{ changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( + password_sessions::login, + password_sessions::current, + password_sessions::logout, + password_sessions::change_password, changes::subscribe, location_health::report, location_health::recent, @@ -91,6 +96,67 @@ pub struct DevelopmentScope { pub triggers: bool, } +#[derive(Debug, Default)] +pub struct ApiAuthentication { + pub development_scope: Option, + pub password_origin: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct PasswordOrigin(pub(crate) String); + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct PasswordLoginRequest { + pub(crate) username: String, + pub(crate) password: String, +} + +impl std::fmt::Debug for PasswordLoginRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PasswordLoginRequest") + .field("username", &self.username) + .finish_non_exhaustive() + } +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct PasswordChangeRequest { + pub(crate) current_password: String, + pub(crate) new_password: String, +} + +impl std::fmt::Debug for PasswordChangeRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PasswordChangeRequest") + .finish_non_exhaustive() + } +} + +#[derive(Serialize, Deserialize, utoipa::ToSchema)] +pub struct PasswordSessionResponse { + pub identity: crate::credentials::models::SessionIdentity, + pub csrf_token: String, +} + +impl std::fmt::Debug for PasswordSessionResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PasswordSessionResponse") + .field("identity", &self.identity) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct AuthenticationDenial { + pub error: String, + pub decision_id: uuid::Uuid, +} + #[derive(Clone, Debug)] pub(crate) struct ChangeFeed { pub(crate) sender: tokio::sync::broadcast::Sender, diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index 5eea412ac1..6e7bae5bf7 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1,2 +1,5 @@ pub(crate) mod locations; +pub(crate) mod password_sessions; +pub(crate) mod passwords; pub(crate) mod scopes; +pub(crate) mod secrets; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs new file mode 100644 index 0000000000..7acd5d5a5c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs @@ -0,0 +1,69 @@ +//! Session creation rechecks the verified password version under the control-store write lock. + +use chrono::{Duration, SubsecRound}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::constants::PASSWORD_SESSION_SECONDS; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, IssuedSession, SessionIdentity, VerifiedPassword}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn issue_session( + store: &ControlStore, + context: &AuditContext, + proof: VerifiedPassword, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT p.principal_id FROM sqb_principals p JOIN sqb_password_credentials c ON p.principal_id=c.principal_id \ + WHERE p.principal_id=$1 AND c.credential_id=$2 AND p.kind='person' AND p.disabled_at IS NULL", + vec![Parameter::Id(proof.principal_id), Parameter::Id(proof.credential_id)]).await?; + if rows.is_empty() { + commit(transaction).await?; + return Ok(None); + } + let session_id = Uuid::now_v7(); + let bearer = new_bearer(session_id)?; + let csrf = new_bearer(session_id)?; + let identity = SessionIdentity { + session_id, + principal_id: proof.principal_id, + expires_at: (now + Duration::seconds(PASSWORD_SESSION_SECONDS)).trunc_subsecs(6), + }; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_password_sessions (session_id, principal_id, credential_id, secret_hash, csrf_hash, created_at, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![Parameter::Id(session_id), Parameter::Id(proof.principal_id), Parameter::Id(proof.credential_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), Parameter::Text(sha256_hex(csrf.as_bytes())), + Parameter::Time(now), Parameter::Time(identity.expires_at)]).await?; + let context = AuditContext { + actor: format!("principal:{}", proof.principal_id), + granting_role: "password-session".into(), + ..context.clone() + }; + commit( + append_audit( + transaction, + &context, + (session_id, "session.created"), + ( + json!({"principal_id": proof.principal_id}), + json!(null), + json!(&identity), + ), + ) + .await?, + ) + .await?; + Ok(Some(IssuedSession { + identity, + bearer, + csrf, + })) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs new file mode 100644 index 0000000000..01a77c91bf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs @@ -0,0 +1,55 @@ +//! Memory-hard password work runs outside the async executor and outside write transactions. + +use argon2::Argon2; +use argon2::password_hash::{Error, PasswordHash, PasswordHasher, PasswordVerifier, SaltString}; + +use crate::credentials::constants::{PASSWORD_ALGORITHM, PASSWORD_KDF_SLOTS}; +use crate::store::errors::ControlError; + +static PASSWORD_WORK: tokio::sync::Semaphore = + tokio::sync::Semaphore::const_new(PASSWORD_KDF_SLOTS); + +pub(crate) async fn hash_password(password: String) -> Result { + let permit = PASSWORD_WORK + .acquire() + .await + .map_err(|_| ControlError::Invalid("password workers are unavailable".into()))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let mut random = [0_u8; 16]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a password salt".into()))?; + let salt = SaltString::encode_b64(&random) + .map_err(|_| ControlError::Invalid("could not encode a password salt".into()))?; + Argon2::default() + .hash_password(password.as_bytes(), &salt) + .map(|hash| hash.to_string()) + .map_err(|_| ControlError::Invalid("could not hash the password".into())) + }) + .await? +} + +pub(crate) async fn verify_hash(hash: String, password: String) -> Result { + let permit = PASSWORD_WORK + .acquire() + .await + .map_err(|_| ControlError::Invalid("password workers are unavailable".into()))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let hash = PasswordHash::new(&hash) + .map_err(|_| ControlError::Invalid("stored password credential is invalid".into()))?; + if hash.algorithm.as_str() != PASSWORD_ALGORITHM { + return Err(ControlError::Invalid( + "stored password credential is not Argon2id".into(), + )); + } + match Argon2::default().verify_password(password.as_bytes(), &hash) { + Ok(()) => Ok(true), + Err(Error::Password) => Ok(false), + Err(_) => Err(ControlError::Invalid( + "stored password credential could not be verified".into(), + )), + } + }) + .await? +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs new file mode 100644 index 0000000000..e0e05f0827 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs @@ -0,0 +1,43 @@ +//! One encoding and constant-time check for high-entropy bearer secrets. + +use std::fmt::Write; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use subtle::ConstantTimeEq; +use uuid::Uuid; + +use crate::credentials::constants::{TOKEN_SECRET_BYTES, TOKEN_SECRET_HEX_LENGTH}; +use crate::store::errors::ControlError; + +pub(crate) fn new_bearer(id: Uuid) -> Result { + let mut random = [0_u8; TOKEN_SECRET_BYTES]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a secure token".into()))?; + let mut bearer = format!("{id}."); + for byte in random { + write!(bearer, "{byte:02x}") + .map_err(|_| ControlError::Invalid("could not format a secure token".into()))?; + } + Ok(bearer) +} + +pub(crate) fn bearer_id(bearer: &str) -> Option { + let (id, secret) = bearer.split_once('.')?; + if secret.len() != TOKEN_SECRET_HEX_LENGTH + || !secret.bytes().all(|value| value.is_ascii_hexdigit()) + { + return None; + } + let Ok(id) = Uuid::parse_str(id) else { + return None; + }; + Some(id) +} + +pub(crate) fn secret_matches(secret: &str, expected_hash: &str) -> bool { + bool::from( + sha256_hex(secret.as_bytes()) + .as_bytes() + .ct_eq(expected_hash.as_bytes()), + ) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs index ce3675812c..d21a3bfd42 100644 --- a/crates/sqlbuild-orchestration/src/credentials/constants.rs +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -2,3 +2,15 @@ pub(crate) const TOKEN_SECRET_BYTES: usize = 32; pub(crate) const TOKEN_SECRET_HEX_LENGTH: usize = TOKEN_SECRET_BYTES * 2; + +pub(crate) const MIN_PASSWORD_CHARACTERS: usize = 12; +pub(crate) const MAX_PASSWORD_BYTES: usize = 1024; +pub(crate) const PASSWORD_ALGORITHM: &str = "argon2id"; +pub(crate) const PASSWORD_KDF_SLOTS: usize = 4; +pub(crate) const PASSWORD_SESSION_SECONDS: i64 = 3600; +pub(crate) const LOGIN_WINDOW_SECONDS: i64 = 60; +pub(crate) const LOGIN_USER_LIMIT: u64 = 5; +pub(crate) const LOGIN_PEER_LIMIT: u64 = 20; +pub(crate) const LOGIN_USERNAME_BYTES: usize = 256; +/// A noncredential PHC output makes missing-user checks perform the same Argon2id work. +pub(crate) const DUMMY_PASSWORD_HASH: &str = "$argon2id$v=19$m=19456,t=2,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs new file mode 100644 index 0000000000..d9fa231835 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs @@ -0,0 +1,36 @@ +//! Live password versions invalidate all earlier sessions without per-process caches. + +use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; +use crate::credentials::models::{SessionAccess, SessionIdentity, StoredSession}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn authenticate_session( + store: &ControlStore, + bearer: &str, + access: SessionAccess<'_>, +) -> Result, ControlError> { + let Some(id) = bearer_id(bearer) else { + return Ok(None); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.session_id, s.principal_id, s.expires_at, s.secret_hash, s.csrf_hash \ + FROM sqb_password_sessions s JOIN sqb_principals p ON p.principal_id=s.principal_id \ + JOIN sqb_password_credentials c ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ + WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ + AND p.kind='person' AND p.disabled_at IS NULL", + vec![Parameter::Id(id), Parameter::Time(now)]).await?; + commit(transaction).await?; + let Some(row) = rows.into_iter().next() else { + return Ok(None); + }; + let session: StoredSession = decode_row(row)?; + let csrf_matches = match access { + SessionAccess::Read => true, + SessionAccess::Write { csrf } => secret_matches(csrf, &session.csrf_hash), + }; + Ok((secret_matches(bearer, &session.secret_hash) && csrf_matches).then_some(session.identity)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs index 4349c0d08b..c2e8a86845 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs @@ -1,11 +1,7 @@ //! Every request sees revocation and current scope state, including across server replicas. -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use subtle::ConstantTimeEq; -use uuid::Uuid; - use crate::credentials::_helpers::scopes::scope_is_active; -use crate::credentials::constants::TOKEN_SECRET_HEX_LENGTH; +use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; use crate::credentials::models::{StoredToken, TokenIdentity}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -18,15 +14,7 @@ pub async fn authenticate_token( store: &ControlStore, bearer: &str, ) -> Result, ControlError> { - let Some((id, secret)) = bearer.split_once('.') else { - return Ok(None); - }; - if secret.len() != TOKEN_SECRET_HEX_LENGTH - || !secret.bytes().all(|value| value.is_ascii_hexdigit()) - { - return Ok(None); - } - let Ok(id) = Uuid::parse_str(id) else { + let Some(id) = bearer_id(bearer) else { return Ok(None); }; let transaction = begin(store, TransactionMode::Read).await?; @@ -44,8 +32,7 @@ pub async fn authenticate_token( }; let token: StoredToken = serde_json::from_value(serde_json::json!(row)) .map_err(|error| ControlError::Invalid(error.to_string()))?; - let digest = sha256_hex(bearer.as_bytes()); - if !bool::from(digest.as_bytes().ct_eq(token.secret_hash.as_bytes())) { + if !secret_matches(bearer, &token.secret_hash) { commit(transaction).await?; return Ok(None); } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs index 090bf85c66..9c9cbd48f0 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -1,14 +1,12 @@ //! Random bearer secrets are returned once; only their hashes and bounded scopes are durable. -use std::fmt::Write; - use chrono::{DateTime, SubsecRound, Utc}; use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; -use crate::credentials::constants::TOKEN_SECRET_BYTES; +use crate::credentials::_helpers::secrets::new_bearer; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; use crate::store::errors::ControlError; @@ -36,14 +34,7 @@ pub async fn issue_token( return Err(ControlError::Invalid("token scope is not active".into())); } let token_id = Uuid::now_v7(); - let mut random = [0_u8; TOKEN_SECRET_BYTES]; - getrandom::fill(&mut random) - .map_err(|_| ControlError::Invalid("could not generate a secure token".into()))?; - let mut bearer = format!("{token_id}."); - for byte in random { - write!(bearer, "{byte:02x}") - .map_err(|_| ControlError::Invalid("could not format a secure token".into()))?; - } + let bearer = new_bearer(token_id)?; let now = transaction.now; let mut parameters: Vec = vec![ Parameter::Id(token_id), diff --git a/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs new file mode 100644 index 0000000000..e7b9254e82 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs @@ -0,0 +1,62 @@ +//! Password login reserves shared throttle capacity before memory-hard verification. + +use uuid::Uuid; + +use crate::credentials::_helpers::password_sessions::issue_session; +use crate::credentials::constants::{LOGIN_USERNAME_BYTES, MAX_PASSWORD_BYTES}; +use crate::credentials::main::record_authentication_denial::record_authentication_denial; +use crate::credentials::main::reserve_password_attempt::reserve_password_attempt; +use crate::credentials::main::verify_password::verify_password; +use crate::credentials::models::{AuditContext, PasswordAttempt, PasswordLogin}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn login_password( + store: &ControlStore, + context: &AuditContext, + (username, password): (&str, &str), +) -> Result { + let bounded = username.len() <= LOGIN_USERNAME_BYTES && password.len() <= MAX_PASSWORD_BYTES; + let username = if bounded { username } else { "" }; + let (attempt_id, allowed) = + reserve_password_attempt(store, context, PasswordAttempt::Login(username)).await?; + if !allowed { + return Ok(PasswordLogin::Denied { + decision_id: attempt_id, + throttled: true, + }); + } + if bounded { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT principal_id FROM sqb_principals WHERE name=$1", + vec![Parameter::Text(username.into())], + ) + .await?; + commit(transaction).await?; + let principal_id = rows + .first() + .map(|row| serde_json::from_value(row["principal_id"].clone())) + .transpose() + .map_err(|_| ControlError::Invalid("stored person identity is invalid".into()))? + .unwrap_or(Uuid::nil()); + if let Some(proof) = verify_password(store, principal_id, password).await? + && let Some(session) = issue_session(store, context, proof).await? + { + return Ok(PasswordLogin::Authenticated(session)); + } + } + let decision_id = record_authentication_denial( + store, + context, + (Uuid::new_v5(&attempt_id, b"denied"), "invalid_credentials"), + serde_json::json!({"attempt_id": attempt_id}), + ) + .await?; + Ok(PasswordLogin::Denied { + decision_id, + throttled: false, + }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index ad09e37418..1f8f07396a 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,7 +1,14 @@ pub(crate) mod append_audit; +pub mod authenticate_session; pub mod authenticate_token; pub mod create_location; pub mod create_principal; pub mod development_location; pub mod issue_token; +pub mod login_password; +pub mod record_authentication_denial; +pub mod reserve_password_attempt; +pub mod revoke_session; pub mod revoke_token; +pub mod set_password; +pub mod verify_password; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs b/crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs new file mode 100644 index 0000000000..c93861e4af --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs @@ -0,0 +1,33 @@ +//! Authentication denials receive durable decision identities without credential material. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn record_authentication_denial( + store: &ControlStore, + context: &AuditContext, + (decision_id, reason): (Uuid, &str), + scope: Value, +) -> Result { + commit( + append_audit( + begin(store, TransactionMode::Write).await?, + context, + (decision_id, "authentication.denied"), + ( + scope, + json!(null), + json!({"allowed": false, "reason": reason}), + ), + ) + .await?, + ) + .await?; + Ok(decision_id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs b/crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs new file mode 100644 index 0000000000..23fa3711b2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs @@ -0,0 +1,72 @@ +//! Serialized reservations bound concurrent password work across API replicas. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::constants::{LOGIN_PEER_LIMIT, LOGIN_USER_LIMIT, LOGIN_WINDOW_SECONDS}; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, PasswordAttempt}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn reserve_password_attempt( + store: &ControlStore, + context: &AuditContext, + attempt: PasswordAttempt<'_>, +) -> Result<(Uuid, bool), ControlError> { + let identity_key = match attempt { + PasswordAttempt::Login(username) => format!("login:{username}"), + PasswordAttempt::Reauthenticate(id) => format!("reauthenticate:{id}"), + }; + let peer = context + .ip_address + .ok_or_else(|| ControlError::Invalid("login requires a trusted peer address".into()))? + .to_canonical() + .to_string(); + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "DELETE FROM sqb_password_login_attempts WHERE attempt_id IN \ + (SELECT attempt_id FROM sqb_password_login_attempts WHERE attempted_at<$1 ORDER BY attempted_at LIMIT 1000)", + vec![Parameter::Time(now - Duration::hours(1))]).await?; + let (mut transaction, rows) = query(transaction, + "SELECT (SELECT COUNT(*) FROM sqb_password_login_attempts WHERE identity_key=$1 AND attempted_at>$3) AS user_count, \ + (SELECT COUNT(*) FROM sqb_password_login_attempts WHERE peer_address=$2 AND attempted_at>$3) AS peer_count", + vec![Parameter::Text(identity_key.clone()), Parameter::Text(peer.clone()), Parameter::Time(now - Duration::seconds(LOGIN_WINDOW_SECONDS))]).await?; + let allowed = rows[0]["user_count"] + .as_u64() + .is_some_and(|count| count < LOGIN_USER_LIMIT) + && rows[0]["peer_count"] + .as_u64() + .is_some_and(|count| count < LOGIN_PEER_LIMIT); + let id = Uuid::now_v7(); + if allowed { + (transaction, _) = query(transaction, + "INSERT INTO sqb_password_login_attempts (attempt_id, identity_key, peer_address, attempted_at) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(id), Parameter::Text(identity_key.clone()), Parameter::Text(peer), Parameter::Time(now)]).await?; + } + commit( + append_audit( + transaction, + context, + ( + id, + if allowed { + "login.requested" + } else { + "login.throttled" + }, + ), + ( + json!({"identity_key": identity_key}), + json!(null), + json!({"allowed": allowed}), + ), + ) + .await?, + ) + .await?; + Ok((id, allowed)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs new file mode 100644 index 0000000000..f7b5c6818b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs @@ -0,0 +1,51 @@ +//! Logout preserves session history and commits its security audit with revocation. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, SessionIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +pub async fn revoke_session( + store: &ControlStore, + context: &AuditContext, + identity: &SessionIdentity, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "UPDATE sqb_password_sessions SET revoked_at=$1 WHERE session_id=$2 \ + AND principal_id=$3 AND revoked_at IS NULL RETURNING session_id", + vec![ + Parameter::Time(now), + Parameter::Id(identity.session_id), + Parameter::Id(identity.principal_id), + ], + ) + .await?; + if rows.is_empty() { + commit(transaction).await?; + return Ok(false); + } + let mut transaction = append_audit( + transaction, + context, + ( + Uuid::new_v5(&identity.session_id, b"revoked"), + "session.revoked", + ), + ( + json!({"principal_id": identity.principal_id}), + json!({"session_id": identity.session_id, "revoked": false}), + json!({"session_id": identity.session_id, "revoked": true}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs new file mode 100644 index 0000000000..60ac393916 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs @@ -0,0 +1,96 @@ +//! Only active people have passwords; changing one rotates its identity and audits atomically. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::passwords::hash_password; +use crate::credentials::constants::{MAX_PASSWORD_BYTES, MIN_PASSWORD_CHARACTERS}; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, PasswordAuthority}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +/// The trusted caller authorizes password management before invoking this storage operation. +pub async fn set_password( + store: &ControlStore, + context: &AuditContext, + authority: PasswordAuthority, + password: &str, +) -> Result<(), ControlError> { + if password.len() > MAX_PASSWORD_BYTES || password.chars().count() < MIN_PASSWORD_CHARACTERS { + return Err(ControlError::Invalid(format!( + "password must contain at least {MIN_PASSWORD_CHARACTERS} characters and at most {MAX_PASSWORD_BYTES} UTF-8 bytes" + ))); + } + let (principal_id, expected) = match authority { + PasswordAuthority::Operator(id) => (id, None), + PasswordAuthority::Reauthenticated { proof, session_id } => { + (proof.principal_id, Some((proof.credential_id, session_id))) + } + }; + let hash = hash_password(password.into()).await?; + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT c.credential_id FROM sqb_principals p \ + LEFT JOIN sqb_password_credentials c ON c.principal_id=p.principal_id \ + WHERE p.principal_id=$1 AND p.kind='person' AND p.disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "password requires an active person".into(), + )); + } + if let Some((id, session_id)) = expected { + let now = transaction.now; + let sessions; + (transaction, sessions) = query( + transaction, + "SELECT session_id FROM sqb_password_sessions WHERE session_id=$1 AND principal_id=$2 \ + AND credential_id=$3 AND revoked_at IS NULL AND expires_at>$4", + vec![ + Parameter::Id(session_id), + Parameter::Id(principal_id), + Parameter::Id(id), + Parameter::Time(now), + ], + ) + .await?; + if rows[0]["credential_id"] != json!(id) || sessions.is_empty() { + return Err(ControlError::Invalid( + "password credential or session changed; authenticate again".into(), + )); + } + } + let before = if rows[0]["credential_id"].is_null() { + json!(null) + } else { + json!({"credential_id": rows[0]["credential_id"]}) + }; + let credential_id = Uuid::now_v7(); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_password_credentials (principal_id, credential_id, password_hash, changed_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (principal_id) DO UPDATE \ + SET credential_id=excluded.credential_id, password_hash=excluded.password_hash, changed_at=excluded.changed_at", + vec![Parameter::Id(principal_id), Parameter::Id(credential_id), Parameter::Text(hash), Parameter::Time(now)], + ) + .await?; + let mut transaction = append_audit( + transaction, + context, + (credential_id, "password.changed"), + ( + json!({"principal_id": principal_id}), + before, + json!({"credential_id": credential_id}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/verify_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/verify_password.rs new file mode 100644 index 0000000000..e2917814b5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/verify_password.rs @@ -0,0 +1,51 @@ +//! A password proves one credential version; it never directly grants a role or an API token. + +use uuid::Uuid; + +use crate::credentials::_helpers::passwords::verify_hash; +use crate::credentials::constants::{DUMMY_PASSWORD_HASH, MAX_PASSWORD_BYTES}; +use crate::credentials::models::VerifiedPassword; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Network callers must apply their shared login throttle before invoking this primitive. +pub async fn verify_password( + store: &ControlStore, + principal_id: Uuid, + password: &str, +) -> Result, ControlError> { + if password.len() > MAX_PASSWORD_BYTES { + return Ok(None); + } + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT credential_id, password_hash FROM sqb_password_credentials c \ + JOIN sqb_principals p ON p.principal_id=c.principal_id \ + WHERE p.principal_id=$1 AND p.kind='person' AND p.disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + commit(transaction).await?; + let row = rows.into_iter().next(); + let (identity, hash) = if let Some(row) = row { + let credential_id = serde_json::from_value(row["credential_id"].clone()) + .map_err(|_| ControlError::Invalid("stored password identity is invalid".into()))?; + let hash = serde_json::from_value(row["password_hash"].clone()) + .map_err(|_| ControlError::Invalid("stored password hash is invalid".into()))?; + ( + Some(VerifiedPassword { + principal_id, + credential_id, + }), + hash, + ) + } else { + (None, DUMMY_PASSWORD_HASH.into()) + }; + Ok(verify_hash(hash, password.into()) + .await? + .then_some(identity) + .flatten()) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index f32ea97224..6ac4e3d876 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -45,6 +45,70 @@ pub struct TokenIdentity { pub expires_at: DateTime, } +/// Proof of one password version, not authorization or an issued session. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VerifiedPassword { + pub principal_id: Uuid, + pub credential_id: Uuid, +} + +#[derive(Clone, Copy, Debug)] +pub enum PasswordAuthority { + Operator(Uuid), + Reauthenticated { + proof: VerifiedPassword, + session_id: Uuid, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SessionIdentity { + pub session_id: Uuid, + pub principal_id: Uuid, + pub expires_at: DateTime, +} + +pub enum SessionAccess<'a> { + Read, + Write { csrf: &'a str }, +} + +pub struct IssuedSession { + pub identity: SessionIdentity, + pub bearer: String, + pub csrf: String, +} + +impl std::fmt::Debug for IssuedSession { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IssuedSession") + .field("identity", &self.identity) + .field("bearer", &"[redacted]") + .field("csrf", &"[redacted]") + .finish() + } +} + +#[derive(Debug)] +pub enum PasswordLogin { + Authenticated(IssuedSession), + Denied { decision_id: Uuid, throttled: bool }, +} + +pub enum PasswordAttempt<'a> { + Login(&'a str), + Reauthenticate(Uuid), +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredSession { + #[serde(flatten)] + pub(crate) identity: SessionIdentity, + pub(crate) secret_hash: String, + pub(crate) csrf_hash: String, +} + /// The bearer is returned once. Debug output must never contain it. #[derive(Serialize)] pub struct IssuedToken { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index b233c2fc4c..9fe48e17cc 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -18,6 +18,8 @@ pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshnes pub(crate) const WAREHOUSE_FINGERPRINTS: &str = include_str!("../warehouse_fingerprints.sql"); pub(crate) const DEVELOPMENT_LOCATIONS: &str = include_str!("../development_locations.sql"); pub(crate) const NODE_TRANSITIONS: &str = include_str!("../node_transitions.sql"); +pub(crate) const PASSWORD_CREDENTIALS: &str = include_str!("../password_credentials.sql"); +pub(crate) const PASSWORD_SESSIONS: &str = include_str!("../password_sessions.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 4927f777f4..c918b577dd 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, - NODE_TRANSITIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, render, verify_migrations, + NODE_TRANSITIONS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, + SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -41,6 +41,8 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_fingerprints", WAREHOUSE_FINGERPRINTS), ("development_locations", DEVELOPMENT_LOCATIONS), ("node_transitions", NODE_TRANSITIONS), + ("password_credentials", PASSWORD_CREDENTIALS), + ("password_sessions", PASSWORD_SESSIONS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/password_credentials.sql b/crates/sqlbuild-orchestration/src/store/password_credentials.sql new file mode 100644 index 0000000000..e881877caf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/password_credentials.sql @@ -0,0 +1,6 @@ +CREATE TABLE sqb_password_credentials ( + principal_id {uuid} PRIMARY KEY REFERENCES sqb_principals(principal_id), + credential_id {uuid} NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + changed_at {time} NOT NULL +); diff --git a/crates/sqlbuild-orchestration/src/store/password_sessions.sql b/crates/sqlbuild-orchestration/src/store/password_sessions.sql new file mode 100644 index 0000000000..77eac4bc8f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/password_sessions.sql @@ -0,0 +1,20 @@ +CREATE TABLE sqb_password_sessions ( + session_id {uuid} PRIMARY KEY, + principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), + credential_id {uuid} NOT NULL, + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time} +); +CREATE INDEX sqb_password_sessions_person ON sqb_password_sessions(principal_id); +CREATE TABLE sqb_password_login_attempts ( + attempt_id {uuid} PRIMARY KEY, + identity_key TEXT NOT NULL, + peer_address TEXT NOT NULL, + attempted_at {time} NOT NULL +); +CREATE INDEX sqb_password_login_user_time ON sqb_password_login_attempts(identity_key, attempted_at); +CREATE INDEX sqb_password_login_peer_time ON sqb_password_login_attempts(peer_address, attempted_at); +CREATE INDEX sqb_password_login_time ON sqb_password_login_attempts(attempted_at); diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index a927f62e60..8fca2ac42a 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -1,17 +1,17 @@ use std::time::Duration; use crate::api::main::serve_api::serve_api; -use crate::api::models::DevelopmentScope; +use crate::api::models::ApiAuthentication; use crate::api::tests::test_types::TestApi; use crate::store::models::ControlStore; pub(crate) async fn start_api(store: &ControlStore) -> TestApi { - start_api_with_scope(store, None).await + start_api_with_authentication(store, ApiAuthentication::default()).await } -pub(crate) async fn start_api_with_scope( +pub(crate) async fn start_api_with_authentication( store: &ControlStore, - scope: Option, + authentication: ApiAuthentication, ) -> TestApi { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await @@ -19,7 +19,7 @@ pub(crate) async fn start_api_with_scope( let url = format!("http://{}", listener.local_addr().expect("API address")); let (shutdown, signal) = tokio::sync::oneshot::channel(); let store = store.clone(); - let task = tokio::spawn(serve_api(store, listener, scope, async { + let task = tokio::spawn(serve_api(store, listener, authentication, async { let _ = signal.await; })); let client = reqwest::Client::builder() diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 615e3cd663..d69d358ad5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -18,6 +18,8 @@ mod test_development; mod test_location_health; #[path = "test_location_work.rs"] mod test_location_work; +#[path = "test_passwords.rs"] +mod test_passwords; #[path = "test_reads.rs"] mod test_reads; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_development.rs b/crates/sqlbuild-orchestration/tests/api/test_development.rs index 11f5256375..3540299f62 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_development.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_development.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::api::models::DevelopmentScope; +use crate::api::models::{ApiAuthentication, DevelopmentScope}; use crate::credentials::main::development_location::development_location; use crate::credentials::models::AuditContext; use crate::declarations::main::{ @@ -15,7 +15,7 @@ use crate::snapshots::main::{ }; use crate::store::main::migrate_store::migrate_store; -use crate::api::tests::helpers::{start_api_with_scope, stop_api}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::DevelopmentTestCase; use crate::credentials::tests::helpers::location_state; use crate::scheduler::tests::contracts::ticks::helpers::snapshot; @@ -82,13 +82,16 @@ fn given_development_when_defaults_publish_then_identity_audit_and_trigger_pause .await .expect("target-specific identity"); assert_ne!(location, other); - let api = start_api_with_scope( + let api = start_api_with_authentication( store, - Some(DevelopmentScope { - location_id: location, - target: "development".into(), - triggers: false, - }), + ApiAuthentication { + development_scope: Some(DevelopmentScope { + location_id: location, + target: "development".into(), + triggers: false, + }), + ..ApiAuthentication::default() + }, ) .await; let client = reqwest::Client::new(); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index d4bcb34632..db9e62620e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -46,16 +46,20 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable let store = &fixture.store; migrate_store(store).await.expect("fresh migration"); // Exercise both deployed predecessors, including checksums and repeated startup. - execute(store, "DROP TABLE sqb_development_locations").await; - execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; - execute(store, "DROP TABLE sqb_warehouse_freshness").await; - execute(store, "DROP INDEX sqb_node_decisive_history; DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; - migrate_store(store).await.expect("schema 2 upgrade"); - execute(store, "DROP TABLE sqb_development_locations").await; - execute(store, "DROP TABLE sqb_warehouse_fingerprints").await; - execute(store, "DROP TABLE sqb_warehouse_freshness").await; - execute(store, "DROP INDEX sqb_node_decisive_history; DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; - migrate_store(store).await.expect("schema 1 upgrade"); + for (version, location_cleanup) in [(3, ""), (2, "DROP TABLE sqb_location_instances;")] + { + execute(store, &format!( + "DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ + DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; \ + DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ + DROP INDEX sqb_node_decisive_history; DROP INDEX sqb_node_finished_cursor; \ + DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ + DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; \ + DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ + {location_cleanup} DELETE FROM sqb_schema_migrations WHERE version>={version}" + )).await; + migrate_store(store).await.expect("predecessor upgrade"); + } migrate_store(store).await.expect("repeat migration"); let location_id = create_location(store, "orders", "test-admin") .await diff --git a/crates/sqlbuild-orchestration/tests/api/test_passwords.rs b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs new file mode 100644 index 0000000000..b961f68ade --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs @@ -0,0 +1,349 @@ +//! Real HTTP sessions use secure cookies, exact origins and live cross-replica credentials. + +use reqwest::header::{COOKIE, HOST, ORIGIN, SET_COOKIE}; +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, AuthenticationDenial, PasswordSessionResponse}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::PasswordApiTestCase; +use crate::credentials::main::{create_principal::create_principal, set_password::set_password}; +use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_password_http_sessions_when_changed_or_revoked_then_cookie_and_csrf_boundaries_hold_across_replicas() + { + let test_cases = [ + PasswordApiTestCase { + description: "SQLite browser session contract", + backend: Backend::Sqlite, + expected_lifetime: 3600, + }, + PasswordApiTestCase { + description: "PostgreSQL browser session contract", + backend: Backend::Postgres, + expected_lifetime: 3600, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/password_credentials.json", + ); + let cases: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop fixture")) + .expect("session inputs"); + let username = cases["username"].as_str().expect("username"); + let maximum_password_bytes = cases["maximum_password_bytes"] + .as_u64() + .expect("password byte limit") as usize; + let password = cases["http_password_character"] + .as_str() + .expect("password character") + .repeat(maximum_password_bytes); + let replacement = cases["http_replacement_character"] + .as_str() + .expect("replacement character") + .repeat(maximum_password_bytes); + let password = password.as_str(); + let replacement = replacement.as_str(); + let origin = cases["origin"].as_str().expect("origin"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = create_principal(store, &context, (username, PrincipalKind::Person)) + .await + .expect("person"); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("set password"); + let api = start_api_with_authentication( + store, + ApiAuthentication { + password_origin: Some(origin.into()), + ..ApiAuthentication::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + password_origin: Some(origin.into()), + ..ApiAuthentication::default() + }, + ) + .await; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + HOST, + origin + .strip_prefix("https://") + .expect("HTTPS origin") + .parse() + .expect("host"), + ); + headers.insert(ORIGIN, origin.parse().expect("origin")); + let client = Client::builder() + .default_headers(headers) + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("browser transport"); + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": username, "password": "x".repeat(16 * 1024)})) + .send() + .await + .expect("oversized request"); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + let mut decisions = Vec::new(); + for invalid_origin in cases["invalid_origins"] + .as_array() + .expect("invalid origins") + { + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .header(ORIGIN, invalid_origin.as_str().expect("origin")) + .json(&json!({"username": username, "password": password})) + .send() + .await + .expect("origin probe"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{}", + test_case.description + ); + decisions.push( + response + .json::() + .await + .expect("denial decision") + .decision_id, + ); + } + for (name, secret) in [ + (username, cases["wrong"].as_str().expect("wrong password")), + ("unknown-person", password), + ] { + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": name, "password": secret})) + .send() + .await + .expect("credential probe"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let denial: AuthenticationDenial = response.json().await.expect("generic denial"); + assert_eq!(denial.error, "authentication rejected"); + decisions.push(denial.decision_id); + } + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": username, "password": password})) + .send() + .await + .expect("login"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let cookies: Vec<_> = response + .headers() + .get_all(SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie").to_owned()) + .collect(); + assert_eq!(cookies.len(), 2); + let session = cookies + .iter() + .find(|cookie| cookie.starts_with("__Host-sqb-session=")) + .expect("session cookie"); + assert!(session.contains("; HttpOnly")); + for cookie in &cookies { + assert!(cookie.contains("; Secure; SameSite=Strict;")); + assert!(cookie.contains("; Path=/;")); + assert!(cookie.contains(&format!("Max-Age={}", test_case.expected_lifetime))); + assert!(!cookie.to_lowercase().contains("domain=")); + } + let cookie = session.split(';').next().expect("cookie pair").to_owned(); + let bearer = cookie.split_once('=').expect("session bearer").1; + let body: Value = response.json().await.expect("login body"); + assert!(body.get("bearer").is_none()); + assert!(!body.to_string().contains(bearer)); + let issued: PasswordSessionResponse = + serde_json::from_value(body).expect("typed session"); + assert_eq!(issued.identity.principal_id, person); + assert!(!format!("{issued:?}").contains(&issued.csrf_token)); + let forged = format!( + "__Host-sqb-session={}.{}", + issued.identity.session_id, + "0".repeat(64) + ); + for invalid_cookie in [&forged, &format!("{cookie}; {cookie}")] { + let response = client + .get(format!("{}/api/auth/session", api.url)) + .header(COOKIE, invalid_cookie) + .send() + .await + .expect("invalid session cookie"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("session decision") + .decision_id, + ); + } + let response = client + .get(format!("{}/api/auth/session", other.url)) + .header(COOKIE, &cookie) + .send() + .await + .expect("second replica read"); + assert_eq!(response.status(), StatusCode::OK); + for csrf in cases["invalid_csrf"].as_array().expect("invalid CSRF") { + let response = client + .post(format!("{}/api/auth/logout", other.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", csrf.as_str().expect("CSRF value")) + .send() + .await + .expect("CSRF probe"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("CSRF decision") + .decision_id, + ); + } + let response = client + .get(format!( + "{}/api/locations/{}/targets/development/runs", + api.url, + Uuid::now_v7() + )) + .header(COOKIE, &cookie) + .send() + .await + .expect("session is not location authority"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = client + .post(format!("{}/api/auth/password", other.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", &issued.csrf_token) + .json(&json!({"current_password": password, "new_password": replacement})) + .send() + .await + .expect("password replacement"); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + for value in response.headers().get_all(SET_COOKIE) { + assert!( + value + .to_str() + .expect("expired cookie") + .contains("Max-Age=0") + ); + } + for server in [&api, &other] { + let response = client + .get(format!("{}/api/auth/session", server.url)) + .header(COOKIE, &cookie) + .send() + .await + .expect("old session check"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("revocation decision") + .decision_id, + ); + } + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": username, "password": replacement})) + .send() + .await + .expect("replacement login"); + assert_eq!(response.status(), StatusCode::OK); + let cookie = response + .headers() + .get_all(SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("new session") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let renewed: PasswordSessionResponse = + response.json().await.expect("new session identity"); + assert_ne!(renewed.identity.session_id, issued.identity.session_id); + let response = client + .post(format!("{}/api/auth/logout", other.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", &renewed.csrf_token) + .send() + .await + .expect("logout"); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + let response = client + .get(format!("{}/api/auth/session", api.url)) + .header(COOKIE, &cookie) + .send() + .await + .expect("logout across replicas"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("logout decision") + .decision_id, + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("audit read"), + "SELECT event_id, action, ip_address, scope_json, before_json, after_json FROM sqb_security_audit", vec![]).await.expect("security evidence"); + commit(transaction).await.expect("audit read complete"); + for decision in decisions { + assert!(rows.iter().any(|row| row["event_id"] == json!(decision) + && row["action"] == "authentication.denied" + && row["ip_address"] == "127.0.0.1")); + } + let audit = serde_json::to_string(&rows).expect("audit"); + for secret in [password, replacement, bearer, &issued.csrf_token] { + assert!( + !audit.contains(secret), + "security audits exclude secret material" + ); + } + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 088a8ac099..2efde86eea 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -74,3 +74,9 @@ pub(crate) struct WarehouseMetadataTestCase { pub(crate) backend: Backend, pub(crate) expected_observations: usize, } + +pub(crate) struct PasswordApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_lifetime: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 9c7936a7ac..1e2ac49d11 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -21,13 +21,18 @@ pub(crate) async fn fingerprint_history( (client, freshness_url, bearer): (&Client, &str, &str), ) { // Upgrade the immediate predecessor while published snapshots and freshness facts exist. - let (transaction, _) = query( - begin(store, TransactionMode::Write).await.expect("upgrade"), - "DROP INDEX sqb_node_decisive_history", - vec![], - ) - .await - .expect("remove subsequent transition migration"); + let mut transaction = begin(store, TransactionMode::Write).await.expect("upgrade"); + for table in ["sqb_password_sessions", "sqb_password_login_attempts"] { + (transaction, _) = query(transaction, &format!("DROP TABLE {table}"), vec![]) + .await + .expect("remove subsequent session migration"); + } + let (transaction, _) = query(transaction, "DROP TABLE sqb_password_credentials", vec![]) + .await + .expect("remove subsequent password migration"); + let (transaction, _) = query(transaction, "DROP INDEX sqb_node_decisive_history", vec![]) + .await + .expect("remove subsequent transition migration"); let (transaction, _) = query(transaction, "DROP TABLE sqb_development_locations", vec![]) .await .expect("remove subsequent development migration"); @@ -36,7 +41,7 @@ pub(crate) async fn fingerprint_history( .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs new file mode 100644 index 0000000000..79b823e572 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs @@ -0,0 +1,286 @@ +//! Real-store password replacement, live eligibility and transactional security audit. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::{ + create_principal::create_principal, set_password::set_password, + verify_password::verify_password, +}; +use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +use crate::credentials::tests::test_passwords::test_types::PasswordTestCase; + +#[path = "test_types.rs"] +mod test_types; + +#[test] +fn given_person_passwords_when_changed_then_replicas_verify_exact_bytes_and_audit_failure_rolls_back() + { + let test_cases = [ + PasswordTestCase { + description: "SQLite password lifecycle", + backend: Backend::Sqlite, + expected_audits: 5, + }, + PasswordTestCase { + description: "PostgreSQL password lifecycle", + backend: Backend::Postgres, + expected_audits: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("fresh schema"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/password_credentials.json", + ); + let cases: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop fixture")) + .expect("password cases"); + let password = cases["password"].as_str().expect("password"); + let replacement = cases["replacement"].as_str().expect("replacement"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = + create_principal(store, &context, ("orders-reader", PrincipalKind::Person)) + .await + .expect("person"); + let service = create_principal( + store, + &context, + ("orders-service", PrincipalKind::ServiceAccount), + ) + .await + .expect("service account"); + for id in [person, service, Uuid::now_v7()] { + assert!( + verify_password(&replica, id, password) + .await + .expect("missing credential") + .is_none() + ); + } + for weak in [ + "", + "12345678901", + cases["too_short"].as_str().expect("short Unicode"), + ] { + assert!( + set_password(store, &context, PasswordAuthority::Operator(person), weak) + .await + .is_err() + ); + } + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(person), + &"x".repeat(1025) + ) + .await + .is_err() + ); + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(service), + password + ) + .await + .is_err() + ); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("set password"); + let first = verify_password(&replica, person, password) + .await + .expect("verify on second store") + .expect("password proof"); + assert_eq!(first.principal_id, person, "{}", test_case.description); + let transaction = begin(&replica, TransactionMode::Read) + .await + .expect("inspect storage"); + let (transaction, rows) = query( + transaction, + "SELECT password_hash FROM sqb_password_credentials WHERE principal_id=$1", + vec![Parameter::Id(person)], + ) + .await + .expect("stored PHC"); + commit(transaction).await.expect("inspection done"); + let first_hash = rows[0]["password_hash"].as_str().expect("PHC").to_owned(); + assert!(first_hash.starts_with("$argon2id$v=19$m=19456,t=2,p=1$")); + assert!(!first_hash.contains(password)); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("same bytes, new salt"); + let second = verify_password(&replica, person, password) + .await + .expect("new version") + .expect("valid password"); + assert_ne!(first.credential_id, second.credential_id); + let (transaction, rows) = query( + begin(&replica, TransactionMode::Read).await.expect("read"), + "SELECT password_hash FROM sqb_password_credentials WHERE principal_id=$1", + vec![Parameter::Id(person)], + ) + .await + .expect("replacement salt"); + commit(transaction).await.expect("read done"); + assert_ne!( + rows[0]["password_hash"], + json!(first_hash), + "identical passwords receive independent salts" + ); + let fault = TransactionFault { + description: "password audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + verify_password(&replica, person, password) + .await + .expect("rolled back"), + Some(second) + ); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement, + ) + .await + .expect("replace password"); + for invalid in [ + password, + cases["wrong"].as_str().expect("wrong"), + &"x".repeat(1025), + ] { + assert!( + verify_password(&replica, person, invalid) + .await + .expect("rejected password") + .is_none() + ); + } + assert!( + verify_password(&replica, person, replacement) + .await + .expect("new password") + .is_some() + ); + for boundary in [ + cases["minimum_unicode"] + .as_str() + .expect("12 Unicode characters"), + &"x".repeat(1024), + ] { + set_password( + store, + &context, + PasswordAuthority::Operator(person), + boundary, + ) + .await + .expect("accepted boundary"); + assert!( + verify_password(&replica, person, boundary) + .await + .expect("boundary verified") + .is_some() + ); + } + migrate_store(&replica) + .await + .expect("restart preserves credentials"); + let (transaction, rows) = query(begin(&replica, TransactionMode::Read).await.expect("audit read"), + "SELECT action, before_json, after_json FROM sqb_security_audit WHERE action='password.changed'", vec![]) + .await.expect("durable audits"); + commit(transaction).await.expect("audit read done"); + assert_eq!( + rows.len(), + test_case.expected_audits, + "failed and rejected changes leave no audit or credential change" + ); + let audit = serde_json::to_string(&rows).expect("audit inspection"); + for secret in [password, replacement, &first_hash] { + assert!( + !audit.contains(secret), + "audit contains identities, never passwords or hashes" + ); + } + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("disable"), + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(chrono::Utc::now()), Parameter::Id(person)], + ) + .await + .expect("disabled person"); + commit(transaction).await.expect("disable committed"); + assert!( + verify_password(&replica, person, &"x".repeat(1024)) + .await + .expect("live eligibility") + .is_none() + ); + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement + ) + .await + .is_err() + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs new file mode 100644 index 0000000000..0398f3ab72 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs @@ -0,0 +1,7 @@ +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct PasswordTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_audits: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs new file mode 100644 index 0000000000..dd888e0ffd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs @@ -0,0 +1,377 @@ +//! Concurrent login reservations, secret eligibility and atomic audit failures on both stores. + +use chrono::Duration; +use futures_util::future::join_all; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::{ + authenticate_session::authenticate_session, authenticate_token::authenticate_token, + create_principal::create_principal, login_password::login_password, + reserve_password_attempt::reserve_password_attempt, revoke_session::revoke_session, + set_password::set_password, verify_password::verify_password, +}; +use crate::credentials::models::{ + AuditContext, PasswordAttempt, PasswordAuthority, PasswordLogin, PrincipalKind, SessionAccess, +}; +use crate::credentials::tests::test_sessions::test_types::SessionTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[path = "test_types.rs"] +mod test_types; + +#[test] +fn given_password_sessions_when_replicas_race_or_auditing_fails_then_throttles_and_credentials_remain_authoritative() + { + let test_cases = [ + SessionTestCase { + description: "SQLite session storage contract", + backend: Backend::Sqlite, + expected_sessions: 5, + expected_peer_attempts: 20, + }, + SessionTestCase { + description: "PostgreSQL session storage contract", + backend: Backend::Postgres, + expected_sessions: 5, + expected_peer_attempts: 20, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/password_credentials.json", + ); + let cases: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop fixture")) + .expect("password inputs"); + let password = cases["password"].as_str().expect("password"); + let replacement = cases["replacement"].as_str().expect("replacement"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("peer")), + }; + let person = + create_principal(store, &context, ("orders-reader", PrincipalKind::Person)) + .await + .expect("person"); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("password"); + let context = &context; + let stores = [store, &replica]; + let outcomes = join_all((0..12).map(|index| async move { + login_password(stores[index % 2], context, ("orders-reader", password)) + .await + .expect("concurrent login") + })) + .await; + assert_eq!( + outcomes + .iter() + .filter(|result| matches!(result, PasswordLogin::Authenticated(_))) + .count(), + test_case.expected_sessions, + "{}", + test_case.description + ); + assert_eq!( + outcomes + .iter() + .filter(|result| matches!( + result, + PasswordLogin::Denied { + throttled: true, + .. + } + )) + .count(), + 12 - test_case.expected_sessions + ); + let retained = outcomes + .into_iter() + .find(|result| matches!(result, PasswordLogin::Authenticated(_))) + .expect("successful session"); + let PasswordLogin::Authenticated(session) = retained else { + panic!("expected an authenticated session"); + }; + for secret in [&session.bearer, &session.csrf] { + assert!(!format!("{session:?}").contains(secret)); + } + assert!( + authenticate_token(&replica, &session.bearer) + .await + .expect("distinct bearer scope") + .is_none() + ); + assert_eq!( + authenticate_session(&replica, &session.bearer, SessionAccess::Read) + .await + .expect("live session"), + Some(session.identity.clone()) + ); + let another_peer = AuditContext { + ip_address: Some("127.0.0.2".parse().expect("other peer")), + ..context.clone() + }; + assert!(matches!( + login_password(&replica, &another_peer, ("orders-reader", password)) + .await + .expect("global identity throttle"), + PasswordLogin::Denied { + throttled: true, + .. + } + )); + let peer_results = join_all((0..30).map(|index| { + let name = format!("customers-{index}"); + let context = &another_peer; + async move { + reserve_password_attempt( + stores[index % 2], + context, + PasswordAttempt::Login(&name), + ) + .await + .expect("shared peer reservation") + .1 + } + })) + .await; + assert_eq!( + peer_results.iter().filter(|allowed| **allowed).count(), + test_case.expected_peer_attempts + ); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("inspect sessions"), + "SELECT secret_hash, csrf_hash FROM sqb_password_sessions", + vec![], + ) + .await + .expect("persisted sessions"); + commit(transaction).await.expect("read complete"); + assert_eq!(rows.len(), test_case.expected_sessions); + let stored = serde_json::to_string(&rows).expect("stored secret hashes"); + assert!(!stored.contains(&session.bearer)); + assert!(!stored.contains(&session.csrf)); + let customer = + create_principal(store, context, ("customers-reader", PrincipalKind::Person)) + .await + .expect("second person"); + set_password( + store, + context, + PasswordAuthority::Operator(customer), + password, + ) + .await + .expect("second password"); + let creation_fault = TransactionFault { + description: "session audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "WHEN NEW.action='session.created'", + sqlite_body: "SELECT RAISE(ABORT,'injected session audit failure')", + postgres_body: "RAISE EXCEPTION 'injected session audit failure'", + }; + install_fault(store, &creation_fault).await; + assert!( + login_password(&replica, context, ("customers-reader", password)) + .await + .is_err() + ); + remove_fault(store, &creation_fault).await; + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("inspect rollback"), + "SELECT session_id FROM sqb_password_sessions WHERE principal_id=$1", + vec![Parameter::Id(customer)], + ) + .await + .expect("no unaudited session"); + commit(transaction).await.expect("read complete"); + assert!(rows.is_empty()); + let PasswordLogin::Authenticated(customer_session) = + login_password(store, context, ("customers-reader", password)) + .await + .expect("customer login") + else { + panic!("expected customer session"); + }; + let revocation_fault = TransactionFault { + description: "logout audit failure", + condition: "WHEN NEW.action='session.revoked'", + ..creation_fault + }; + install_fault(store, &revocation_fault).await; + assert!( + revoke_session(store, context, &customer_session.identity) + .await + .is_err() + ); + remove_fault(store, &revocation_fault).await; + assert!( + authenticate_session(&replica, &customer_session.bearer, SessionAccess::Read) + .await + .expect("failed logout rolls back") + .is_some() + ); + let proof = verify_password(store, customer, password) + .await + .expect("verify version") + .expect("password proof"); + set_password( + &replica, + context, + PasswordAuthority::Operator(customer), + replacement, + ) + .await + .expect("operator reset"); + assert!( + authenticate_session(store, &customer_session.bearer, SessionAccess::Read) + .await + .expect("reset invalidates session") + .is_none() + ); + assert!( + set_password( + store, + context, + PasswordAuthority::Reauthenticated { + proof, + session_id: customer_session.identity.session_id + }, + password + ) + .await + .is_err() + ); + let proof = verify_password(store, customer, replacement) + .await + .expect("new version") + .expect("replacement proof"); + assert!( + set_password( + store, + context, + PasswordAuthority::Reauthenticated { + proof, + session_id: customer_session.identity.session_id + }, + password + ) + .await + .is_err() + ); + assert!( + revoke_session(store, context, &customer_session.identity) + .await + .expect("logout") + ); + assert!( + !revoke_session(&replica, context, &customer_session.identity) + .await + .expect("idempotent logout") + ); + let PasswordLogin::Authenticated(disabled_session) = + login_password(store, context, ("customers-reader", replacement)) + .await + .expect("login before disabling person") + else { + panic!("expected live session before disabling person"); + }; + let transaction = begin(&replica, TransactionMode::Write) + .await + .expect("disable person"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(now), Parameter::Id(customer)], + ) + .await + .expect("disabled person"); + commit(transaction).await.expect("disable saved"); + assert!( + authenticate_session(store, &disabled_session.bearer, SessionAccess::Read) + .await + .expect("disabled person's session rejected across replicas") + .is_none() + ); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("expire session"); + let expired = transaction.now - Duration::seconds(1); + let (transaction, _) = query( + transaction, + "UPDATE sqb_password_sessions SET expires_at=$1 WHERE session_id=$2", + vec![ + Parameter::Time(expired), + Parameter::Id(session.identity.session_id), + ], + ) + .await + .expect("expired session"); + commit(transaction).await.expect("expiry saved"); + assert!( + authenticate_session(&replica, &session.bearer, SessionAccess::Read) + .await + .expect("expired session rejected") + .is_none() + ); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("session history"), + "SELECT revoked_at FROM sqb_password_sessions WHERE session_id=$1", + vec![Parameter::Id(customer_session.identity.session_id)], + ) + .await + .expect("revoked history retained"); + commit(transaction).await.expect("history complete"); + assert!(!rows[0]["revoked_at"].is_null()); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("audit history"), + "SELECT action, after_json FROM sqb_security_audit WHERE event_id=$1", + vec![Parameter::Id(Uuid::new_v5( + &customer_session.identity.session_id, + b"revoked", + ))], + ) + .await + .expect("one revocation audit"); + commit(transaction).await.expect("audit complete"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["action"], json!("session.revoked")); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs new file mode 100644 index 0000000000..7892e45278 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs @@ -0,0 +1,8 @@ +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct SessionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_sessions: usize, + pub(crate) expected_peer_attempts: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index edb65f4446..7bf01dda3a 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -6,5 +6,9 @@ pub(crate) mod helpers; pub(crate) mod location_helpers; #[path = "test_contract.rs"] mod test_contract; +#[path = "passwords/test_contract.rs"] +mod test_passwords; +#[path = "sessions/test_contract.rs"] +mod test_sessions; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 947bcd1c60..89b7861714 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -422,9 +422,18 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r ) .await; // Recreate the exact pre-change index/migration state with populated run history. - let transaction = begin(store, TransactionMode::Write) + let mut transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); + for table in ["sqb_password_sessions", "sqb_password_login_attempts"] { + (transaction, _) = query(transaction, &format!("DROP TABLE {table}"), vec![]) + .await + .expect("remove subsequent session migration"); + } + let (transaction, _) = + query(transaction, "DROP TABLE sqb_password_credentials", vec![]) + .await + .expect("remove subsequent password migration"); let (transaction, _) = query(transaction, "DROP INDEX sqb_node_decisive_history", vec![]) .await diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index a6f3b0dbe3..61c612feda 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ + "DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ + DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_decisive_history; \ DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ diff --git a/fensu.toml b/fensu.toml index 70ff31ff00..51d4478e3c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1567,8 +1567,14 @@ reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot imp [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/credentials/models.rs" -symbols = ["IssuedToken"] -reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." +symbols = ["IssuedToken", "IssuedSession"] +reason = "Issued credentials implement Debug manually to redact bearer and CSRF secrets; deriving Debug would expose them." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/api/models.rs" +symbols = ["PasswordLoginRequest", "PasswordChangeRequest", "PasswordSessionResponse"] +reason = "Password transport models implement Debug manually to redact passwords and CSRF secrets; other models derive Debug." [[targets.rust.rule_exceptions]] rule = "FPRSS201" From 7908e6013236844d6f0aa4325f55e62a812251b3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:39:12 +0000 Subject: [PATCH 069/138] feat: advance shared authentication revision atomically Couple existing credential invalidations to an additive shared revision in the same transaction. Verify rollback, idempotent retries and separate-process visibility on SQLite and PostgreSQL without introducing an authentication cache or new authority. --- .../orchestration/password_credentials.json | 1 + .../tests/native_commands/admin/test_admin.rs | 9 ++- .../src/store/_helpers/changes.rs | 6 ++ .../src/store/_helpers/migration.rs | 1 + .../src/store/authentication_revision.sql | 5 ++ .../src/store/main/migrate_store.rs | 8 ++- .../tests/api/test_location_work.rs | 3 +- .../tests/api/warehouse_metadata/helpers.rs | 8 ++- .../tests/credentials/helpers.rs | 21 +++++++ .../credentials/passwords/test_contract.rs | 18 ++++++ .../tests/credentials/test_contract.rs | 61 +++++++++++++++---- .../tests/sensors/test_runs.rs | 6 +- .../tests/snapshots/helpers.rs | 3 +- 13 files changed, 128 insertions(+), 22 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/store/authentication_revision.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json b/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json index 0f690df069..d84e84d2d0 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json +++ b/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json @@ -9,6 +9,7 @@ "maximum_password_bytes": 1024, "http_password_character": "\u0000", "http_replacement_character": "\u0001", + "security_revision_after_replacement": 3, "invalid_origins": ["https://other.example.com", "null"], "invalid_csrf": ["", "not-a-session-secret"] } diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index cb22985e20..e2e17a2675 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -139,10 +139,17 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope let token_id = credential["identity"]["token_id"] .as_str() .expect("credential ID"); - for expected in [true, false] { + let connection = rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("parent observes shared revision"); + let read_revision = || connection.query_row( + "SELECT revision FROM sqb_authentication_revision WHERE singleton=1", [], + |row| row.get::<_, u64>(0)).expect("cross-process revision"); + let revision = read_revision(); + for (expected, delta) in [(true, 1), (false, 1)] { let revoked = administer(&control, &["revoke-token", token_id]); assert_eq!(revoked["revoked"], expected); assert_eq!(revoked["token_id"], token_id); + assert_eq!(read_revision(), revision + delta); } assert!(matches!( publish_remote_snapshot(&url, bearer, &snapshot).await, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs index cd23dd0052..424c3ad917 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs @@ -39,6 +39,12 @@ pub(crate) async fn resolve_changes( let mut messages: BTreeSet = BTreeSet::new(); for change in std::mem::take(&mut transaction.changes) { if change == Change::Credentials { + (transaction, _) = query( + transaction, + "UPDATE sqb_authentication_revision SET revision=revision+1 WHERE singleton=1", + vec![], + ) + .await?; messages.insert(ChangeMessage::CredentialsChanged); continue; } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 9fe48e17cc..94306d2b83 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -20,6 +20,7 @@ pub(crate) const DEVELOPMENT_LOCATIONS: &str = include_str!("../development_loca pub(crate) const NODE_TRANSITIONS: &str = include_str!("../node_transitions.sql"); pub(crate) const PASSWORD_CREDENTIALS: &str = include_str!("../password_credentials.sql"); pub(crate) const PASSWORD_SESSIONS: &str = include_str!("../password_sessions.sql"); +pub(crate) const AUTHENTICATION_REVISION: &str = include_str!("../authentication_revision.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/authentication_revision.sql b/crates/sqlbuild-orchestration/src/store/authentication_revision.sql new file mode 100644 index 0000000000..d753945d49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/authentication_revision.sql @@ -0,0 +1,5 @@ +CREATE TABLE sqb_authentication_revision ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + revision BIGINT NOT NULL CHECK (revision >= 0) +); +INSERT INTO sqb_authentication_revision (singleton, revision) VALUES (1, 0); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index c918b577dd..dc1fc3e4d8 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,9 +4,10 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, - NODE_TRANSITIONS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, - SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, + AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, + LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, + render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -43,6 +44,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("node_transitions", NODE_TRANSITIONS), ("password_credentials", PASSWORD_CREDENTIALS), ("password_sessions", PASSWORD_SESSIONS), + ("authentication_revision", AUTHENTICATION_REVISION), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index db9e62620e..9686e6f45e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -49,7 +49,8 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable for (version, location_cleanup) in [(3, ""), (2, "DROP TABLE sqb_location_instances;")] { execute(store, &format!( - "DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ + "DROP TABLE sqb_authentication_revision; \ + DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; \ DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_decisive_history; DROP INDEX sqb_node_finished_cursor; \ diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 1e2ac49d11..b7f7d2bafd 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -22,7 +22,11 @@ pub(crate) async fn fingerprint_history( ) { // Upgrade the immediate predecessor while published snapshots and freshness facts exist. let mut transaction = begin(store, TransactionMode::Write).await.expect("upgrade"); - for table in ["sqb_password_sessions", "sqb_password_login_attempts"] { + for table in [ + "sqb_authentication_revision", + "sqb_password_sessions", + "sqb_password_login_attempts", + ] { (transaction, _) = query(transaction, &format!("DROP TABLE {table}"), vec![]) .await .expect("remove subsequent session migration"); @@ -41,7 +45,7 @@ pub(crate) async fn fingerprint_history( .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index 4c28e90101..66fa0571ae 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -37,3 +37,24 @@ pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { .await .expect("read durable token hashes") } + +pub(crate) async fn authentication_revision( + store: &ControlStore, +) -> Result { + use crate::store::main::{begin::begin, commit::commit, query::query}; + use crate::store::models::TransactionMode; + + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + "SELECT revision FROM sqb_authentication_revision WHERE singleton=1", + vec![], + ) + .await?; + commit(transaction).await?; + rows.first() + .and_then(|row| row.get("revision")) + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| { + crate::store::errors::ControlError::Schema("authentication revision is invalid".into()) + }) +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs index 79b823e572..12dc42386d 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs @@ -8,6 +8,7 @@ use crate::credentials::main::{ verify_password::verify_password, }; use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; +use crate::credentials::tests::helpers::authentication_revision; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, }; @@ -171,6 +172,9 @@ fn given_person_passwords_when_changed_then_replicas_verify_exact_bytes_and_audi sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; + let revision = authentication_revision(&replica) + .await + .expect("password revision"); install_fault(store, &fault).await; assert!( set_password( @@ -183,6 +187,12 @@ fn given_person_passwords_when_changed_then_replicas_verify_exact_bytes_and_audi .is_err() ); remove_fault(store, &fault).await; + assert_eq!( + authentication_revision(&replica) + .await + .expect("audit rollback revision"), + revision + ); assert_eq!( verify_password(&replica, person, password) .await @@ -197,6 +207,14 @@ fn given_person_passwords_when_changed_then_replicas_verify_exact_bytes_and_audi ) .await .expect("replace password"); + assert_eq!( + authentication_revision(&replica) + .await + .expect("replacement revision"), + cases["security_revision_after_replacement"] + .as_u64() + .expect("expected revision") + ); for invalid in [ password, cases["wrong"].as_str().expect("wrong"), diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 4e1e3252ac..54d77e1e96 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -13,7 +13,7 @@ use crate::store::main::migrate_store::migrate_store; use chrono::{Duration, Utc}; use uuid::Uuid; -use crate::credentials::tests::helpers::{location_state, stored_hashes}; +use crate::credentials::tests::helpers::{authentication_revision, location_state, stored_hashes}; use crate::credentials::tests::test_types::{CredentialTestCase, TransactionFault}; use crate::runs::tests::helpers::{queue, request, snapshot}; use crate::runs::tests::terminal_helpers::finish; @@ -47,6 +47,12 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the let store = &fixture.store; migrate_store(store).await.expect("migrate"); let replica = second_store(store).await; + assert_eq!( + authentication_revision(&replica) + .await + .expect("initial revision"), + 0 + ); let context = AuditContext { actor: "test-admin".into(), granting_role: "admin".into(), @@ -101,6 +107,12 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; + let revision_fault = TransactionFault { + description: "authentication revision failure", + table: "sqb_authentication_revision", + operation: "UPDATE", + ..fault + }; install_fault(store, &fault).await; assert!( create_principal( @@ -160,29 +172,52 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .is_none() ); } - install_fault(store, &fault).await; + let revision = authentication_revision(&replica) + .await + .expect("before revocation"); + for failure in [&fault, &revision_fault] { + install_fault(store, failure).await; + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, failure).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("failed revocation rolled back") + .is_some() + ); + assert_eq!( + authentication_revision(&replica) + .await + .expect("rollback revision"), + revision + ); + } assert!( revoke_token(store, &context, issued.identity.token_id) .await - .is_err() - ); - remove_fault(store, &fault).await; - assert!( - authenticate_token(&replica, &issued.bearer) - .await - .expect("failed revocation rolled back") - .is_some() + .expect("revoke") ); - assert!( - revoke_token(store, &context, issued.identity.token_id) + assert_eq!( + authentication_revision(&replica) .await - .expect("revoke") + .expect("committed revision"), + revision + 1 ); assert!( !revoke_token(&replica, &context, issued.identity.token_id) .await .expect("retry revoke") ); + assert_eq!( + authentication_revision(store) + .await + .expect("idempotent revision"), + revision + 1 + ); assert!( authenticate_token(&replica, &issued.bearer) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 89b7861714..9eec7fbb3e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -425,7 +425,11 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r let mut transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); - for table in ["sqb_password_sessions", "sqb_password_login_attempts"] { + for table in [ + "sqb_authentication_revision", + "sqb_password_sessions", + "sqb_password_login_attempts", + ] { (transaction, _) = query(transaction, &format!("DROP TABLE {table}"), vec![]) .await .expect("remove subsequent session migration"); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 61c612feda..96b8b3d1b1 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ + "DROP TABLE sqb_authentication_revision; \ + DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_decisive_history; \ DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ From 47f7f6c70e3e88ac6f10f55e4d139986b63d8196 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:27:11 +0000 Subject: [PATCH 070/138] feat: expose native password provisioning and server configuration Enable explicitly configured browser password sessions and trusted local owner password provisioning without adding role or machine authority. Read secrets from hidden confirmed terminal input or bounded standard input, and cover standalone reset, invalidation and configuration rejection. --- Cargo.lock | 22 +++ Cargo.toml | 1 + crates/sqlbuild-cli/Cargo.toml | 1 + .../sqlbuild-cli/src/argv/command_table.json | 24 +++ .../src/native_commands/_helpers/decision.rs | 1 + .../src/native_commands/_helpers/terminal.rs | 38 ++++- .../src/native_commands/constants.rs | 2 + .../src/native_commands/main/_run_admin.rs | 15 +- .../src/native_commands/main/_run_control.rs | 7 +- .../src/native_commands/models.rs | 1 + crates/sqlbuild-cli/tests/native_commands.rs | 2 + .../tests/native_commands/admin/helpers.rs | 44 +++-- .../native_commands/admin/test_passwords.rs | 152 ++++++++++++++++++ .../tests/native_commands/helpers.rs | 30 ++-- .../tests/native_commands/test_control.rs | 20 +++ .../tests/native_commands/test_types.rs | 14 ++ .../tests/native_commands/worker/helpers.rs | 17 +- .../native_commands/worker/test_recovery.rs | 8 +- 18 files changed, 350 insertions(+), 49 deletions(-) create mode 100644 crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs diff --git a/Cargo.lock b/Cargo.lock index 839ea3ccd7..97e9dbad01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2120,6 +2120,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rpassword" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da316a15f47e3d053de9cb2c439650bd8fa4aaeb9365f2e5f27f492ff73c196" +dependencies = [ + "libc", + "rtoolbox", + "windows-sys 0.61.2", +] + +[[package]] +name = "rtoolbox" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a1efe12a1469752d0e6ff5ebec0b6ef4924cc5c4c71046b0ec730040535819d" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "ruff_python_ast" version = "0.0.4" @@ -2546,6 +2567,7 @@ dependencies = [ "ctrlc", "getrandom 0.4.3", "reqwest", + "rpassword", "rusqlite", "rustix", "same-file", diff --git a/Cargo.toml b/Cargo.toml index d6680590b1..567c8c800f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } +rpassword = "7.5.4" polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index cac9dd214d..6e46d7a345 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -20,6 +20,7 @@ blake3.workspace = true chrono.workspace = true getrandom.workspace = true reqwest.workspace = true +rpassword.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } sqlbuild-build-features.workspace = true diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index dfd5c016a0..8fa6940051 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -328,6 +328,18 @@ ] } }, + { + "name": "set-password", + "parser": { + "prog": "sqb admin set-password", + "description": "Set or reset an active person's password through direct control-store ownership. Reads hidden, confirmed terminal input or one line from standard input; never grants roles.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "PRINCIPAL_ID", "required": true, "help": "active person UUID; password is read from the terminal or standard input, never an argument", "group": 0} + ] + } + }, { "name": "issue-location-token", "parser": { @@ -894,6 +906,11 @@ "metavar": "create-principal", "help": "create a person or service account without granting permissions" }, + { + "dest": "set-password", + "metavar": "set-password", + "help": "set an active person's password without granting permissions" + }, { "dest": "issue-location-token", "metavar": "issue-location-token", @@ -1010,6 +1027,13 @@ "help": "TCP port, or 0 to choose a free port (default: 8000)", "group": 1 }, + { + "kind": "store", + "options": ["--password-origin"], + "dest": "control_password_origin", + "help": "enable password sessions for this exact external HTTPS origin; requires HTTPS termination in front of the HTTP listener", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index f78d6f1874..112cb865c0 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -225,6 +225,7 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result Ok(ControlAdmin::RevokeToken(id)), + Some("set-password") => Ok(ControlAdmin::SetPassword(id)), _ => Err(ControlError::Invalid( "an administration command is required".into(), )), diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs index cd354451ed..5476f82e40 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -1,12 +1,15 @@ //! The process's terminal: its width, colour support and stream writes. -use std::io::{self, IsTerminal, Write}; +use std::io::{self, BufRead, IsTerminal, Read, Write}; + +use sqlbuild_orchestration::store::errors::ControlError; use crate::argv::models::Stream; use crate::native_commands::_helpers::argv_host::python_int; use crate::native_commands::constants::{ BROKEN_PIPE_EXCEPTION, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, EXIT_FLUSH_FAILURE_STATUS, - FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PYTHON_STDOUT_BUFFER_SIZE, UNCAUGHT_EXCEPTION_STATUS, + FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PASSWORD_INPUT_BYTES, PYTHON_STDOUT_BUFFER_SIZE, + UNCAUGHT_EXCEPTION_STATUS, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::LostStdout; @@ -54,6 +57,37 @@ pub(crate) fn stdout_supports_color() -> bool { std::env::var_os(NO_COLOR_VARIABLE).is_none() && io::stdout().is_terminal() } +/// Passwords never enter parsed arguments, environment variables or command output. +pub(crate) fn read_password() -> Result { + let password = if io::stdin().is_terminal() { + let password = rpassword::prompt_password("Password: ")?; + if password != rpassword::prompt_password("Confirm password: ")? { + return Err(ControlError::Invalid("passwords do not match".into())); + } + password + } else { + let mut password = String::new(); + io::stdin() + .lock() + .take(PASSWORD_INPUT_BYTES) + .read_line(&mut password)?; + if password.len() as u64 == PASSWORD_INPUT_BYTES { + return Err(ControlError::Invalid("password input is too large".into())); + } + if password.ends_with('\n') { + password.pop(); + if password.ends_with('\r') { + password.pop(); + } + } + password + }; + if password.is_empty() { + return Err(ControlError::Invalid("password input was empty".into())); + } + Ok(password) +} + /// `\n` as a Python text stream writes it on this platform. fn console_bytes(text: &str) -> Vec { if cfg!(windows) { diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 65784a5e9b..e959a4b263 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -93,3 +93,5 @@ pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; /// Reserve half the location work claim for acknowledgement retries after killing a callback. pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; +/// Bound piped terminal input independently of the credential owner's password policy. +pub(crate) const PASSWORD_INPUT_BYTES: u64 = 4096; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 2580c2b194..267371bda7 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -5,7 +5,8 @@ use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::credentials::main::set_password::set_password; +use sqlbuild_orchestration::credentials::models::{AuditContext, PasswordAuthority}; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; use sqlbuild_orchestration::platform::main::location_work::location_work; @@ -20,6 +21,7 @@ use uuid::Uuid; use crate::argv::models::Namespace; use crate::native_commands::_helpers::decision::control_admin; use crate::native_commands::_helpers::execution::open_control_store; +use crate::native_commands::_helpers::terminal::read_password; use crate::native_commands::models::ControlAdmin; pub(crate) async fn run_admin(namespace: &Namespace) -> Result { @@ -48,6 +50,17 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let password = read_password()?; + set_password( + &store, + &context, + PasswordAuthority::Operator(principal_id), + &password, + ) + .await?; + Ok(json!({"principal_id": principal_id, "password_set": true})) + } ControlAdmin::RequestJobRun(request) => { let applied = request_job_run(&store, &context, &request).await?; Ok(json!({"run_id": request.run_id, "applied": applied})) diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index c557ba3bce..4c737efd2d 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -9,7 +9,7 @@ use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, flag}; +use crate::native_commands::_helpers::decision::{control_address, flag, text}; use crate::native_commands::_helpers::execution::{ open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; @@ -46,7 +46,10 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - ApiAuthentication::default(), + ApiAuthentication { + password_origin: text(namespace, "control_password_origin"), + ..ApiAuthentication::default() + }, stopped(shutdown.clone()), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index b16504cefa..67f2a4af34 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -19,6 +19,7 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + SetPassword(Uuid), RequestJobRun(JobRunRequest), Rerun(RerunRequest), RequestLocationWork(LocationWorkRequest), diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index 208f02150b..1258cd6628 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -6,6 +6,8 @@ mod helpers; mod test_admin; #[path = "native_commands/test_control.rs"] mod test_control; +#[path = "native_commands/admin/test_passwords.rs"] +mod test_passwords; #[path = "native_commands/test_types.rs"] mod test_types; #[path = "native_commands/worker/test_worker.rs"] diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs index d43fbfa744..4820f96f28 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -1,6 +1,7 @@ //! Run the isolated native executable; never include returned bearer bytes in assertion messages. -use std::process::{Command, Output}; +use std::io::Write; +use std::process::{Command, Output, Stdio}; use serde_json::Value; use sqlbuild_orchestration::declarations::models::TriggerRecord; @@ -23,19 +24,42 @@ pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { } pub(crate) fn invoke_admin(control: &TestControl, arguments: &[&str]) -> Output { - Command::new( + admin_command(control, arguments) + .output() + .expect("native administration process") +} + +fn admin_command(control: &TestControl, arguments: &[&str]) -> Command { + let mut command = Command::new( control .directory .path() .join(format!("sqb{}", std::env::consts::EXE_SUFFIX)), - ) - .arg("admin") - .args(arguments) - .current_dir(control.directory.path()) - .env("PATH", "") - .env_remove("SQLBUILD_CONTROL_DATABASE_URL") - .output() - .expect("native administration process") + ); + command + .arg("admin") + .args(arguments) + .current_dir(control.directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL"); + command +} + +pub(crate) fn invoke_password(control: &TestControl, principal: &str, input: &[u8]) -> Output { + let mut child = admin_command(control, &["set-password", principal]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("native password provisioning"); + let mut stdin = child.stdin.take().expect("password pipe"); + stdin + .write_all(input) + .expect("deliver password through stdin"); + drop(stdin); + child + .wait_with_output() + .expect("password command completed") } pub(crate) async fn read_triggers( diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs new file mode 100644 index 0000000000..37e9631d84 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs @@ -0,0 +1,152 @@ +//! Standalone owner provisioning enables browser identity, never machine-route authority. + +use std::time::Duration; + +use reqwest::{StatusCode, header}; +use serde_json::{Value, json}; + +use crate::admin_helpers::{administer, invoke_password}; +use crate::helpers::{api_ready, start_control, wait_for_exit}; +use crate::test_types::{PasswordCliTestCase, PasswordOriginTestCase}; + +#[test] +fn given_owner_provisioned_passwords_when_a_standalone_server_authenticates_then_resets_invalidate_sessions_without_granting_authority() + { + let test_cases = [PasswordCliTestCase { + description: "owner-provisioned identity remains separate from location authority", + password: " orders-password-café ", + replacement: "customers-password-2026", + expected_password_rows: 1, + expected_audits: 2, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler", "--password-origin", "https://control.example.com"]); + let url = api_ready(&mut control).await; + let person = administer(&control, &["create-principal", "orders-reader"]); + let principal = person["principal_id"].as_str().expect("person identity"); + let service = administer(&control, &["create-principal", "orders-service", "--kind", "service_account"]); + let password = test_case.password; + let invalid_inputs = [Vec::new(), b"short\n".to_vec(), vec![b'x'; 5000], vec![0xff, b'\n']]; + for input in invalid_inputs { + let output = invoke_password(&control, principal, &input); + assert!(!output.status.success(), "invalid stdin must not set a password"); + assert!(output.stdout.is_empty()); + } + let rejected = invoke_password(&control, service["principal_id"].as_str().expect("service identity"), password.as_bytes()); + assert!(!rejected.status.success()); + assert!(rejected.stdout.is_empty()); + assert!(!String::from_utf8_lossy(&rejected.stderr).contains(password)); + + let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().expect("HTTP client"); + let login = |password: &str| client.post(format!("{url}/api/auth/password/login")) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .json(&json!({"username": "orders-reader", "password": password})).send(); + let output = invoke_password(&control, principal, format!("{password}\r\n").as_bytes()); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let result: Value = serde_json::from_slice(&output.stdout).expect("JSON-only provisioning result"); + assert_eq!(result, json!({"principal_id": principal, "password_set": true})); + assert!(!String::from_utf8_lossy(&output.stderr).contains(password)); + let response = login(password).await.expect("provisioned password login"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + let cookie = response.headers().get_all(header::SET_COOKIE).iter() + .filter_map(|header| header.to_str().ok()) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("host-only session cookie").split(';').next().expect("cookie pair").to_owned(); + let body: Value = response.json().await.expect("session identity"); + assert_eq!(body["identity"]["principal_id"], principal); + let session = client.get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com").header(header::COOKIE, &cookie) + .send().await.expect("current session"); + assert_eq!(session.status(), StatusCode::OK); + let machine = client.get(format!("{url}/api/locations/{}/targets/development/triggers", uuid::Uuid::now_v7())) + .header(header::HOST, "control.example.com").header(header::COOKIE, &cookie) + .send().await.expect("browser cookie has no location authority"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + let reset = invoke_password(&control, principal, test_case.replacement.as_bytes()); + assert!(reset.status.success(), "{}", String::from_utf8_lossy(&reset.stderr)); + assert_eq!(serde_json::from_slice::(&reset.stdout).expect("reset JSON"), json!({"principal_id": principal, "password_set": true})); + assert!(!String::from_utf8_lossy(&reset.stderr).contains(test_case.replacement)); + let previous = client.get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com").header(header::COOKIE, cookie) + .send().await.expect("previous session after child-process reset"); + assert_eq!(previous.status(), StatusCode::UNAUTHORIZED); + assert_eq!(login(password).await.expect("old password rejected").status(), StatusCode::UNAUTHORIZED); + assert_eq!(login(test_case.replacement).await.expect("replacement password accepted").status(), StatusCode::OK); + let connection = rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")).expect("inspect durable provisioning"); + let (count, hash): (i64, String) = connection.query_row( + "SELECT COUNT(*), MIN(password_hash) FROM sqb_password_credentials", [], + |row| Ok((row.get(0)?, row.get(1)?))).expect("password storage"); + assert_eq!(count, test_case.expected_password_rows, "{}", test_case.description); + assert!(hash.starts_with("$argon2id$")); + let audits: i64 = connection.query_row( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='password.changed' AND actor='local-control-store-owner' AND granting_role='control-store-owner'", [], + |row| row.get(0)).expect("owner authority audit"); + assert_eq!(audits, test_case.expected_audits, "{}", test_case.description); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_invalid_password_origins_when_a_standalone_server_starts_then_it_exits_with_a_configuration_error() + { + let test_cases = [ + PasswordOriginTestCase { + description: "HTTP is not an external HTTPS origin", + origin: "http://control.example.com", + expected_error: "password authentication requires an HTTPS origin", + }, + PasswordOriginTestCase { + description: "origin cannot contain a path", + origin: "https://control.example.com/path", + expected_error: "password authentication requires an HTTPS origin", + }, + PasswordOriginTestCase { + description: "origin cannot contain credentials", + origin: "https://user:password@control.example.com", + expected_error: "password authentication requires an HTTPS origin", + }, + PasswordOriginTestCase { + description: "empty origin is not a request to disable authentication", + origin: "", + expected_error: "password authentication requires an HTTPS origin", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let origin = test_case.origin; + let mut control = start_control(&[ + "server", + "--port", + "0", + "--no-scheduler", + "--password-origin", + origin, + ]); + let status = wait_for_exit(&mut control).await; + assert!(!status.success()); + let log = std::fs::read_to_string(&control.stderr).expect("startup error"); + assert!( + log.contains(test_case.expected_error), + "{}", + test_case.description + ); + assert!( + !log.contains(origin) || origin.is_empty(), + "invalid configuration is not echoed" + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index ec2421c381..908b552555 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -1,5 +1,5 @@ use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Command, ExitStatus, Stdio}; use std::time::Duration; use crate::test_types::{ServiceTestCase, TestControl}; @@ -119,18 +119,7 @@ pub(crate) async fn terminate(control: &mut TestControl) { .status() .expect("send SIGTERM"); assert!(status.success()); - let status = tokio::time::timeout(Duration::from_secs(5), async { - for _ in 0..250 { - tokio::time::sleep(Duration::from_millis(20)).await; - let Some(status) = control.child.try_wait().expect("child exit") else { - continue; - }; - return status; - } - panic!("control roles exhausted shutdown attempts"); - }) - .await - .expect("control roles stop promptly"); + let status = wait_for_exit(control).await; assert!( status.success(), "shutdown failed: {}", @@ -142,3 +131,18 @@ pub(crate) async fn terminate(control: &mut TestControl) { .ends_with("Control service stopped successfully\n") ); } + +pub(crate) async fn wait_for_exit(control: &mut TestControl) -> ExitStatus { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let Some(exit) = control.child.try_wait().expect("control process exit") else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return exit; + } + panic!("control process exhausted shutdown attempts"); + }) + .await + .expect("control process exits promptly") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 215540fdb2..59bea6e574 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -28,6 +28,13 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "usage: sqb admin ", expected_stderr: "", }, + CommandTestCase { + description: "native password provisioning help", + arguments: &["admin", "set-password", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin set-password ", + expected_stderr: "", + }, CommandTestCase { description: "native trigger administration help", arguments: &["admin", "change-trigger", "--help"], @@ -179,6 +186,13 @@ fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_ expected_stdout_prefix: "", expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", }, + CommandTestCase { + description: "invalid password principal rejected before input or store initialization", + arguments: &["admin", "set-password", "not-a-uuid"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", + }, CommandTestCase { description: "invalid expiry rejected before store initialization", arguments: &[ @@ -390,6 +404,12 @@ fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migra "{}", test_case.description ); + let password_session = client + .get(format!("{url}/api/auth/session")) + .send() + .await + .expect("password routes are opt-in"); + assert_eq!(password_session.status(), reqwest::StatusCode::NOT_FOUND); let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; assert_eq!( leaders, test_case.expected_leaders, diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 4796bc1277..1d9cd298b0 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -27,6 +27,20 @@ pub(crate) struct ServiceTestCase { pub(crate) expected_heartbeats: i64, } +pub(crate) struct PasswordCliTestCase { + pub(crate) description: &'static str, + pub(crate) password: &'static str, + pub(crate) replacement: &'static str, + pub(crate) expected_password_rows: i64, + pub(crate) expected_audits: i64, +} + +pub(crate) struct PasswordOriginTestCase { + pub(crate) description: &'static str, + pub(crate) origin: &'static str, + pub(crate) expected_error: &'static str, +} + pub(crate) struct TestControl { pub(crate) child: Child, pub(crate) directory: tempfile::TempDir, diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 5a5a7fc4a0..a615b93b6e 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -1,7 +1,7 @@ //! Privileged test operators stay separate from the API-only worker session. use std::collections::BTreeMap; -use std::process::{Command, ExitStatus, Stdio}; +use std::process::{Command, Stdio}; use std::time::{Duration, SystemTime}; use sqlbuild_orchestration::credentials::main::create_location::create_location; @@ -350,18 +350,3 @@ pub(crate) async fn wait_for_worker_log(control: &mut TestControl, expected: &st .await .expect("worker supervision phase completes"); } - -pub(crate) async fn wait_for_worker_exit(control: &mut TestControl) -> ExitStatus { - tokio::time::timeout(Duration::from_secs(5), async { - for _ in 0..250 { - let Some(exit) = control.child.try_wait().expect("scheduler exit") else { - tokio::time::sleep(Duration::from_millis(20)).await; - continue; - }; - return exit; - } - panic!("worker supervision exhausted shutdown attempts"); - }) - .await - .expect("shutdown publishes retained terminal evidence") -} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs index 0a723a1db4..50c375ca27 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -2,10 +2,8 @@ use std::time::Duration; -use crate::helpers::{api_ready, start_control}; -use crate::worker::helpers::{ - claim_worker_run, start_recovery_scheduler, wait_for_worker_exit, wait_for_worker_log, -}; +use crate::helpers::{api_ready, start_control, wait_for_exit}; +use crate::worker::helpers::{claim_worker_run, start_recovery_scheduler, wait_for_worker_log}; use crate::worker::test_types::WorkerRecoveryTestCase; #[test] @@ -74,7 +72,7 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ connection .execute_batch("DROP TRIGGER reject_recovery_publication") .expect("restore terminal publication"); - let exit = wait_for_worker_exit(&mut replacement).await; + let exit = wait_for_exit(&mut replacement).await; assert!( !exit.success(), "the original publication failure remains visible" From 60d29dc7d4240334d5d1eefa36bfd5d11f9321b0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:35:10 +0000 Subject: [PATCH 071/138] test: exercise browser sessions through the generated control client Verify cookie forwarding, CSRF denial, logout invalidation and separation from location authority against the native server. Preserve exact-origin enforcement through a disposable HTTP forwarding fixture; this test does not verify external TLS termination. --- clients/typescript/tests/client.test.ts | 65 +++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/clients/typescript/tests/client.test.ts b/clients/typescript/tests/client.test.ts index 7385be356d..6bff0fd253 100644 --- a/clients/typescript/tests/client.test.ts +++ b/clients/typescript/tests/client.test.ts @@ -5,10 +5,11 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { once } from "node:events"; +import { createServer, request as forwardRequest } from "node:http"; import { test } from "node:test"; import { createControlClient, type components } from "../src/client.ts"; -test("generated client preserves native HTTP scopes, replay, path encoding and SSE", { timeout: 20000 }, async () => { +test("generated client preserves native scopes, replay, path encoding, SSE and browser sessions", { timeout: 20000 }, async () => { const directory = await mkdtemp(join(tmpdir(), "sqlbuild-client-")); const executable = process.env.SQLBUILD_EXECUTABLE ?? "sqb"; const env = { ...process.env }; @@ -17,18 +18,43 @@ test("generated client preserves native HTTP scopes, replay, path encoding and S const admin = (...args: string[]) => JSON.parse(execFileSync(executable, ["admin", ...args], { cwd: directory, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], })); + let baseUrl: string; + // Exercise exact-origin forwarding without asking fetch to override its forbidden Host header. + // This HTTP fixture checks the transport contract, not external TLS termination. + const proxy = createServer((request, response) => { + const forwarded = forwardRequest(`${baseUrl}${request.url}`, { + method: request.method, headers: request.headers, + }, (upstream) => { + response.writeHead(upstream.statusCode!, upstream.headers); + upstream.pipe(response); + }); + forwarded.on("error", () => { response.writeHead(502); response.end(); }); + request.pipe(forwarded); + }); let server: ReturnType | undefined; try { + proxy.listen(0, "127.0.0.1"); + await once(proxy, "listening"); + const address = proxy.address(); + assert.ok(address && typeof address !== "string"); + const browserUrl = `http://127.0.0.1:${address.port}`; + const origin = `https://127.0.0.1:${address.port}`; const { location_id } = admin("create-location", "orders"); const other = admin("create-location", "inventory").location_id; const { bearer } = admin("issue-location-token", location_id, "--expires-at", new Date(Date.now() + 60000).toISOString()); - server = spawn(executable, ["server", "--port", "0", "--no-scheduler"], { + const { principal_id } = admin("create-principal", "orders-reader"); + const password = " orders-password-café "; + execFileSync(executable, ["admin", "set-password", principal_id], { + cwd: directory, env, input: `${password}\n`, stdio: ["pipe", "pipe", "pipe"], + }); + server = spawn(executable, ["server", "--port", "0", "--no-scheduler", + "--password-origin", origin], { cwd: directory, env, stdio: ["ignore", "ignore", "pipe"], }); const running = server; // Readiness is emitted on stderr; never log the one-time credential response. - const baseUrl = await new Promise((resolve, reject) => { + baseUrl = await new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error("API readiness timed out")), 5000); let log = ""; running.once("error", reject); @@ -81,12 +107,45 @@ test("generated client preserves native HTTP scopes, replay, path encoding and S } finally { await reader.cancel(); } + const browser = createControlClient({ baseUrl: browserUrl, headers: { Origin: origin } }); + const login = await browser.POST("/api/auth/password/login", { + body: { username: "orders-reader", password }, + }); + assert.equal(login.response.status, 200); + assert.equal(login.response.headers.get("cache-control"), "no-store"); + assert.equal(login.data?.identity.principal_id, principal_id); + assert.ok(login.data?.csrf_token); + assert.equal("bearer" in login.data, false); + const sessionCookie = login.response.headers.getSetCookie() + .find((cookie) => cookie.startsWith("__Host-sqb-session=")); + assert.ok(sessionCookie); + assert.match(sessionCookie, /; Secure;/); + assert.match(sessionCookie, /; HttpOnly$/); + const headers = { Cookie: sessionCookie.split(";")[0] }; + // The transport does not persist credentials or turn a person's session into location authority. + assert.equal((await browser.GET("/api/auth/session")).response.status, 401); + const session = await browser.GET("/api/auth/session", { headers }); + assert.deepEqual(session.data, login.data.identity); + const machine = await browser.GET(healthPath, { headers, params }); + assert.equal(machine.response.status, 401); + const invalidLogout = await browser.POST("/api/auth/logout", { + headers, params: { header: { "x-sqb-csrf": "incorrect-csrf" } }, + }); + assert.equal(invalidLogout.response.status, 401); + assert.ok(invalidLogout.error?.decision_id); + assert.equal((await browser.GET("/api/auth/session", { headers })).response.status, 200); + const logout = await browser.POST("/api/auth/logout", { + headers, params: { header: { "x-sqb-csrf": login.data.csrf_token } }, + }); + assert.equal(logout.response.status, 204); + assert.equal((await browser.GET("/api/auth/session", { headers })).response.status, 401); } finally { if (server && server.exitCode === null) { const exited = once(server, "exit"); server.kill("SIGTERM"); await exited; } + await new Promise((resolve) => proxy.close(() => resolve())); await rm(directory, { recursive: true, force: true }); } }); From a094855708f2ebffc4321df0751f305bec54ea66 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:53:16 +0000 Subject: [PATCH 072/138] feat: rotate live credentials atomically without widening authority Keep scope and expiry unchanged while replacing a bearer, revoking its predecessor and linking the security audit in one serialized transaction. Reuse issuance and revocation mutations, retain live-scope checks and shared revision invalidation, and verify replica, race, rollback, terminal-run and standalone CLI behavior with synthetic lifecycle fixtures. --- .../orchestration/token_rotation.sql | 37 +++++ .../sqlbuild-cli/src/argv/command_table.json | 17 ++ .../src/native_commands/_helpers/decision.rs | 1 + .../src/native_commands/main/_run_admin.rs | 4 + .../src/native_commands/models.rs | 1 + .../tests/native_commands/admin/test_admin.rs | 15 ++ .../tests/native_commands/admin/test_types.rs | 2 +- .../src/credentials/_helpers/mod.rs | 1 + .../src/credentials/_helpers/tokens.rs | 92 +++++++++++ .../src/credentials/main/issue_token.rs | 60 +------ .../src/credentials/main/mod.rs | 1 + .../src/credentials/main/revoke_token.rs | 33 +--- .../src/credentials/main/rotate_token.rs | 48 ++++++ .../tests/credentials/test_contract.rs | 147 ++++++++++++++++-- 14 files changed, 366 insertions(+), 93 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql b/benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql new file mode 100644 index 0000000000..22bb5fce87 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql @@ -0,0 +1,37 @@ +-- Historical credentials must not regain authority through rotation. +INSERT INTO sqb_principals (principal_id, name, kind, created_at) +VALUES ('00000000-0000-4000-8000-000000000011', 'expired-orders-service', 'service_account', '2020-01-01T00:00:00Z'); +INSERT INTO sqb_principals (principal_id, name, kind, created_at, disabled_at) +VALUES ('00000000-0000-4000-8000-000000000021', 'disabled-inventory-service', 'service_account', '2020-01-01T00:00:00Z', '2020-01-02T00:00:00Z'); +INSERT INTO sqb_access_tokens + (token_id, secret_hash, principal_id, scope_json, created_at, expires_at) +VALUES ( + '00000000-0000-4000-8000-000000000012', + '0000000000000000000000000000000000000000000000000000000000000000', + '00000000-0000-4000-8000-000000000011', + '{"kind":"principal","principal_id":"00000000-0000-4000-8000-000000000011"}', + '2020-01-01T00:00:00Z', + '2020-01-02T00:00:00Z' +); +INSERT INTO sqb_access_tokens + (token_id, secret_hash, principal_id, scope_json, created_at, expires_at) +VALUES ( + '00000000-0000-4000-8000-000000000022', + '0000000000000000000000000000000000000000000000000000000000000000', + '00000000-0000-4000-8000-000000000021', + '{"kind":"principal","principal_id":"00000000-0000-4000-8000-000000000021"}', + '2020-01-01T00:00:00Z', + '2999-01-01T00:00:00Z' +); +INSERT INTO sqb_locations (location_id, name, created_by, created_at, archived_at) +VALUES ('00000000-0000-4000-8000-000000000031', 'archived-orders', 'synthetic-operator', '2020-01-01T00:00:00Z', '2020-01-02T00:00:00Z'); +INSERT INTO sqb_access_tokens + (token_id, secret_hash, location_id, scope_json, created_at, expires_at) +VALUES ( + '00000000-0000-4000-8000-000000000032', + '0000000000000000000000000000000000000000000000000000000000000000', + '00000000-0000-4000-8000-000000000031', + '{"kind":"location","location_id":"00000000-0000-4000-8000-000000000031"}', + '2020-01-01T00:00:00Z', + '2999-01-01T00:00:00Z' +); diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 8fa6940051..0c091e4e3e 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -477,6 +477,18 @@ ] } }, + { + "name": "rotate-token", + "parser": { + "prog": "sqb admin rotate-token", + "description": "Replace a live credential atomically through direct control-store ownership, retaining its exact scope and expiry. Returns the new bearer once as JSON; the old credential immediately stops working.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "TOKEN_ID", "required": true, "help": "live credential UUID, never its bearer secret", "group": 0} + ] + } + }, { "name": "rerun-run", "parser": { @@ -926,6 +938,11 @@ "metavar": "revoke-token", "help": "revoke a credential by UUID" }, + { + "dest": "rotate-token", + "metavar": "rotate-token", + "help": "replace a live credential without changing its scope or expiry" + }, { "dest": "request-run", "metavar": "request-run", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 112cb865c0..8e78095dff 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -225,6 +225,7 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result Ok(ControlAdmin::RevokeToken(id)), + Some("rotate-token") => Ok(ControlAdmin::RotateToken(id)), Some("set-password") => Ok(ControlAdmin::SetPassword(id)), _ => Err(ControlError::Invalid( "an administration command is required".into(), diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 267371bda7..468ad81089 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -5,6 +5,7 @@ use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::main::rotate_token::rotate_token; use sqlbuild_orchestration::credentials::main::set_password::set_password; use sqlbuild_orchestration::credentials::models::{AuditContext, PasswordAuthority}; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; @@ -50,6 +51,9 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + Ok(json!(rotate_token(&store, &context, token_id).await?)) + } ControlAdmin::SetPassword(principal_id) => { let password = read_password()?; set_password( diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 67f2a4af34..d940e6fe67 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -19,6 +19,7 @@ pub(crate) enum ControlAdmin { CreatePrincipal(String, PrincipalKind), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), + RotateToken(Uuid), SetPassword(Uuid), RequestJobRun(JobRunRequest), Rerun(RerunRequest), diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs index e2e17a2675..b2ebaaaee2 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -28,7 +28,10 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope "location.created", "location.created", "token.issued", + "token.issued", + "token.revoked", "token.revoked", + "token.rotated", "trigger.changed", "trigger.changed", "trigger.changed", @@ -102,6 +105,18 @@ fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scope .await .expect("operator-issued credential publishes"); assert_eq!(published.location_id, location_id); + let rotated = administer(&control, &["rotate-token", credential["identity"]["token_id"].as_str().expect("old token UUID")]); + assert_eq!(rotated["identity"]["scope"], credential["identity"]["scope"]); + assert_eq!(rotated["identity"]["expires_at"], credential["identity"]["expires_at"]); + assert_ne!(rotated["identity"]["token_id"], credential["identity"]["token_id"]); + assert_ne!(rotated["bearer"], credential["bearer"]); + assert!(matches!(publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)))); + let repeated = invoke_admin(&control, &["rotate-token", credential["identity"]["token_id"].as_str().expect("old token UUID")]); + assert!(!repeated.status.success()); + assert!(repeated.stdout.is_empty()); + let credential = rotated; + let bearer = credential["bearer"].as_str().expect("rotated one-time bearer"); let trigger_id = read_triggers(&url, bearer, (location_id, "development")).await[0].trigger_id.to_string(); for case in [ TriggerAdminTestCase { description: "enable overrides the paused code default", change: "enable", cursor_arguments: &[], expected_status: "running", expected_cursor: None }, diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs index 8f0c87914a..34a4bf871d 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -2,7 +2,7 @@ pub(crate) struct AdminTestCase { pub(crate) description: &'static str, pub(crate) location_name: &'static str, pub(crate) expected_name: &'static str, - pub(crate) expected_actions: [&'static str; 9], + pub(crate) expected_actions: [&'static str; 12], } pub(crate) struct PrincipalAdminTestCase { diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index 6e7bae5bf7..b6363dfde5 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod password_sessions; pub(crate) mod passwords; pub(crate) mod scopes; pub(crate) mod secrets; +pub(crate) mod tokens; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs new file mode 100644 index 0000000000..d3163cad01 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs @@ -0,0 +1,92 @@ +//! Token mutations compose within the caller's single audited write transaction. + +use chrono::{DateTime, SubsecRound, Utc}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn issue( + transaction: ControlTransaction, + context: &AuditContext, + scope: CredentialScope, + expires_at: DateTime, +) -> Result<(ControlTransaction, IssuedToken), ControlError> { + let expires_at = expires_at.trunc_subsecs(6); + if expires_at <= transaction.now { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + let (transaction, active) = scope_is_active(transaction, scope).await?; + if !active { + return Err(ControlError::Invalid("token scope is not active".into())); + } + let token_id = Uuid::now_v7(); + let bearer = new_bearer(token_id)?; + let now = transaction.now; + let mut parameters = vec![ + Parameter::Id(token_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + ]; + parameters.extend(scope_parameters(scope)); + parameters.extend([ + Parameter::Json(json!(scope)), + Parameter::Time(now), + Parameter::Time(expires_at), + ]); + let (transaction, _) = query(transaction, + "INSERT INTO sqb_access_tokens (token_id, secret_hash, principal_id, location_id, run_id, claim_id, scope_json, created_at, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", parameters).await?; + let identity = TokenIdentity { + token_id, + scope, + expires_at, + }; + let transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"issued"), "token.issued"), + (json!(scope), json!(null), json!(identity)), + ) + .await?; + Ok((transaction, IssuedToken { identity, bearer })) +} + +pub(crate) async fn revoke( + transaction: ControlTransaction, + context: &AuditContext, + token_id: Uuid, +) -> Result<(ControlTransaction, bool), ControlError> { + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_access_tokens SET revoked_at=$1 WHERE token_id=$2 AND revoked_at IS NULL RETURNING scope_json", + vec![Parameter::Time(now), Parameter::Id(token_id)]).await?; + let Some(row) = rows.into_iter().next() else { + return Ok((transaction, false)); + }; + let scope = row + .get("scope_json") + .cloned() + .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; + let mut transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), + ( + scope, + json!({"token_id": token_id, "revoked": false}), + json!({"token_id": token_id, "revoked": true}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs index 9c9cbd48f0..94840e1952 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -1,19 +1,13 @@ //! Random bearer secrets are returned once; only their hashes and bounded scopes are durable. -use chrono::{DateTime, SubsecRound, Utc}; -use serde_json::json; -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use uuid::Uuid; +use chrono::{DateTime, Utc}; -use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; -use crate::credentials::_helpers::secrets::new_bearer; -use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; +use crate::credentials::_helpers::tokens::issue; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// The caller authorizes token management before entering this storage operation. pub async fn issue_token( @@ -22,48 +16,8 @@ pub async fn issue_token( scope: CredentialScope, expires_at: DateTime, ) -> Result { - let expires_at = expires_at.trunc_subsecs(6); let transaction = begin(store, TransactionMode::Write).await?; - if expires_at <= transaction.now { - return Err(ControlError::Invalid( - "token expiry must be in the future".into(), - )); - } - let (transaction, active) = scope_is_active(transaction, scope).await?; - if !active { - return Err(ControlError::Invalid("token scope is not active".into())); - } - let token_id = Uuid::now_v7(); - let bearer = new_bearer(token_id)?; - let now = transaction.now; - let mut parameters: Vec = vec![ - Parameter::Id(token_id), - Parameter::Text(sha256_hex(bearer.as_bytes())), - ]; - parameters.extend(scope_parameters(scope)); - parameters.extend([ - Parameter::Json(json!(scope)), - Parameter::Time(now), - Parameter::Time(expires_at), - ]); - let (transaction, _) = query(transaction, - "INSERT INTO sqb_access_tokens (token_id, secret_hash, principal_id, location_id, run_id, claim_id, scope_json, created_at, expires_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", - parameters).await?; - let identity = TokenIdentity { - token_id, - scope, - expires_at, - }; - commit( - append_audit( - transaction, - context, - (Uuid::new_v5(&token_id, b"issued"), "token.issued"), - (json!(scope), json!(null), json!(identity)), - ) - .await?, - ) - .await?; - Ok(IssuedToken { identity, bearer }) + let (transaction, issued) = issue(transaction, context, scope, expires_at).await?; + commit(transaction).await?; + Ok(issued) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index 1f8f07396a..eb9b124133 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -10,5 +10,6 @@ pub mod record_authentication_denial; pub mod reserve_password_attempt; pub mod revoke_session; pub mod revoke_token; +pub mod rotate_token; pub mod set_password; pub mod verify_password; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs index 76d619d5be..4df9aed7c4 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -1,15 +1,13 @@ //! Revocation and its security audit are one transaction, without a per-process token cache. -use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::_helpers::tokens::revoke; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// Repeating a revocation does not create a second audit entry or rewrite its actor. pub async fn revoke_token( @@ -18,30 +16,7 @@ pub async fn revoke_token( token_id: Uuid, ) -> Result { let transaction = begin(store, TransactionMode::Write).await?; - let now = transaction.now; - let (transaction, rows) = query(transaction, - "UPDATE sqb_access_tokens SET revoked_at=$1 WHERE token_id=$2 AND revoked_at IS NULL RETURNING scope_json", - vec![Parameter::Time(now), Parameter::Id(token_id)]).await?; - let Some(row) = rows.into_iter().next() else { - commit(transaction).await?; - return Ok(false); - }; - let scope = row - .get("scope_json") - .cloned() - .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; - let mut transaction = append_audit( - transaction, - context, - (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), - ( - scope, - json!({"token_id": token_id, "revoked": false}), - json!({"token_id": token_id, "revoked": true}), - ), - ) - .await?; - transaction.changes.insert(Change::Credentials); + let (transaction, revoked) = revoke(transaction, context, token_id).await?; commit(transaction).await?; - Ok(true) + Ok(revoked) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs new file mode 100644 index 0000000000..cc93a374ff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs @@ -0,0 +1,48 @@ +//! Replace one live credential without widening its scope or extending its lifetime. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::tokens::{issue, revoke}; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The trusted caller authorizes token management; the old UUID is not an authentication proof. +pub async fn rotate_token( + store: &ControlStore, + context: &AuditContext, + token_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT token_id, scope_json AS scope, expires_at FROM sqb_access_tokens \ + WHERE token_id=$1 AND revoked_at IS NULL AND expires_at>$2", + vec![Parameter::Id(token_id), Parameter::Time(now)], + ) + .await?; + let before: TokenIdentity = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("token is not active".into()))?, + )?; + let (transaction, _) = revoke(transaction, context, token_id).await?; + let (transaction, issued) = + issue(transaction, context, before.scope, before.expires_at).await?; + let transaction = append_audit( + transaction, + context, + ( + Uuid::new_v5(&issued.identity.token_id, b"rotated"), + "token.rotated", + ), + (json!(before.scope), json!(before), json!(issued.identity)), + ) + .await?; + commit(transaction).await?; + Ok(issued) +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 54d77e1e96..bcb86ea674 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -5,6 +5,7 @@ use crate::credentials::main::create_location::create_location; use crate::credentials::main::create_principal::create_principal; use crate::credentials::main::issue_token::issue_token; use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::main::rotate_token::rotate_token; use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; use crate::runs::main::claim_next_run::claim_next_run; use crate::scheduler::tests::contracts::helpers::acquire_lease; @@ -59,16 +60,19 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the request_id: Uuid::now_v7(), ip_address: Some("127.0.0.1".parse().expect("loopback")), }; - let predecessor = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../benchmarks/duckshop/edge_cases/orchestration/legacy_person_token.sql"); - let sql = std::fs::read_to_string(predecessor).expect("DuckShop predecessor token"); let mut transaction = begin(store, TransactionMode::Write) .await .expect("predecessor"); - for statement in sql.split(';').map(str::trim).filter(|sql| !sql.is_empty()) { - (transaction, _) = query(transaction, statement, vec![]) - .await - .expect("legacy person and hashed token"); + for name in ["legacy_person_token.sql", "token_rotation.sql"] { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration") + .join(name); + let sql = std::fs::read_to_string(path).expect("DuckShop predecessor tokens"); + for statement in sql.split(';').map(str::trim).filter(|sql| !sql.is_empty()) { + (transaction, _) = query(transaction, statement, vec![]) + .await + .expect("synthetic predecessor token"); + } } commit(transaction).await.expect("predecessor committed"); let personal_scope = CredentialScope::Principal { @@ -88,9 +92,27 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the ); assert_eq!( stored_hashes(&replica).await.len(), - 1, + 4, "rejected issuance writes nothing" ); + for id in [ + "00000000-0000-4000-8000-000000000002", + "00000000-0000-4000-8000-000000000012", + "00000000-0000-4000-8000-000000000022", + "00000000-0000-4000-8000-000000000032", + ] { + assert!( + rotate_token( + store, + &context, + Uuid::parse_str(id).expect("synthetic token identity") + ) + .await + .is_err(), + "person, expired, disabled and archived-location credentials cannot be rotated" + ); + } + assert_eq!(stored_hashes(&replica).await.len(), 4); let legacy_bearer = format!("00000000-0000-4000-8000-000000000002.{}", "0".repeat(64)); assert!( authenticate_token(&replica, &legacy_bearer) @@ -113,6 +135,16 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the operation: "UPDATE", ..fault }; + let insertion_fault = TransactionFault { + description: "replacement issuance failure", + table: "sqb_access_tokens", + ..fault + }; + let rotation_audit_fault = TransactionFault { + description: "rotation linkage audit failure", + condition: "WHEN NEW.action='token.rotated'", + ..fault + }; install_fault(store, &fault).await; assert!( create_principal( @@ -124,7 +156,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .is_err() ); assert!(create_location(store, "orders", &context).await.is_err()); - assert_eq!(location_state(&replica).await, (0, 0)); + assert_eq!(location_state(&replica).await, (1, 0)); remove_fault(store, &fault).await; let principal_id = create_principal( store, @@ -136,7 +168,7 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the let location_id = create_location(store, "orders", &context) .await .expect("location"); - assert_eq!(location_state(&replica).await, (1, 1)); + assert_eq!(location_state(&replica).await, (2, 1)); let scopes = [ CredentialScope::Principal { principal_id }, CredentialScope::Location { location_id }, @@ -172,6 +204,83 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the .is_none() ); } + let revision = authentication_revision(&replica) + .await + .expect("before rotation"); + let count = stored_hashes(store).await.len(); + for failure in [ + &fault, + &insertion_fault, + &rotation_audit_fault, + &revision_fault, + ] { + install_fault(store, failure).await; + assert!( + rotate_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, failure).await; + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("old credential survives rollback"), + Some(issued.identity.clone()) + ); + assert_eq!(stored_hashes(&replica).await.len(), count); + assert_eq!( + authentication_revision(&replica) + .await + .expect("rollback revision"), + revision + ); + } + let (first, second) = tokio::join!( + rotate_token(store, &context, issued.identity.token_id), + rotate_token(&replica, &context, issued.identity.token_id), + ); + assert_ne!( + first.is_ok(), + second.is_ok(), + "concurrent rotations have exactly one winner" + ); + let rotated = first.or(second).expect("rotation winner"); + assert_eq!(rotated.identity.scope, issued.identity.scope); + assert_eq!(rotated.identity.expires_at, issued.identity.expires_at); + assert_ne!(rotated.identity.token_id, issued.identity.token_id); + assert_ne!(rotated.bearer, issued.bearer); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("old token revoked atomically") + .is_none() + ); + assert_eq!( + authenticate_token(store, &rotated.bearer) + .await + .expect("replacement authenticates"), + Some(rotated.identity.clone()) + ); + assert_eq!(stored_hashes(&replica).await.len(), count + 1); + assert_eq!( + authentication_revision(&replica) + .await + .expect("one committed rotation"), + revision + 1 + ); + let (transaction, audits) = query(begin(store, TransactionMode::Read).await.expect("read audit"), + "SELECT before_json, after_json FROM sqb_security_audit WHERE action='token.rotated'", vec![]).await.expect("rotation audit"); + commit(transaction).await.expect("audit read"); + let audit = audits + .iter() + .find(|audit| { + audit["before_json"]["token_id"] + == serde_json::json!(issued.identity.token_id) + }) + .expect("linked rotation history"); + assert_eq!(audit["before_json"], serde_json::json!(issued.identity)); + assert_eq!(audit["after_json"], serde_json::json!(rotated.identity)); + let issued = rotated; let revision = authentication_revision(&replica) .await .expect("before revocation"); @@ -279,6 +388,18 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the let issued = issue_token(store, &context, scope, expiry) .await .expect("mint claimed-run token"); + let rotated = rotate_token(store, &context, issued.identity.token_id) + .await + .expect("rotate claimed-run credential"); + assert_eq!(rotated.identity.scope, issued.identity.scope); + assert_eq!(rotated.identity.expires_at, issued.identity.expires_at); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("old run credential revoked") + .is_none() + ); + let issued = rotated; assert_eq!( authenticate_token(&replica, &issued.bearer) .await @@ -309,6 +430,12 @@ fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_the issue_token(store, &context, scope, expiry).await.is_err(), "a terminal claim cannot mint a new token" ); + assert!( + rotate_token(store, &context, issued.identity.token_id) + .await + .is_err(), + "a terminal claim cannot regain authority through rotation" + ); close_store(replica).await; close_test_store(fixture).await; }); From 948f50b38b4138c1cb46fbf96840d487bfb5d444 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:32:34 +0000 Subject: [PATCH 073/138] feat: authenticate explicitly bound people through trusted proxies Require an opt-in HTTPS origin, exact canonical socket peers and explicit upstream-subject bindings. Audit live-person authentication and durable denials without issuing sessions, provisioning accounts or granting machine authority. Cover spoofed headers, replica disablement and standalone configuration failures; retain the password transport contract and generated client parity. --- .../orchestration/proxy_identities.json | 9 + clients/typescript/README.md | 30 ++ clients/typescript/src/schema.d.ts | 48 ++++ .../sqlbuild-cli/src/argv/command_table.json | 7 + .../src/native_commands/_helpers/decision.rs | 21 ++ .../src/native_commands/main/_run_control.rs | 9 +- crates/sqlbuild-cli/tests/native_commands.rs | 2 + .../tests/native_commands/helpers.rs | 4 + .../tests/native_commands/test_proxy.rs | 125 +++++++++ .../tests/native_commands/test_types.rs | 13 + .../api/_helpers/authentication/configure.rs | 14 +- .../src/api/_helpers/authentication/mod.rs | 1 + .../authentication/password_sessions.rs | 27 +- .../authentication/password_transport.rs | 31 ++- .../_helpers/authentication/trusted_proxy.rs | 109 ++++++++ .../src/api/constants.rs | 1 + .../src/api/main/openapi_document.rs | 12 +- .../sqlbuild-orchestration/src/api/models.rs | 19 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_proxy.rs | 261 ++++++++++++++++++ .../tests/api/test_types.rs | 7 + 21 files changed, 718 insertions(+), 34 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json create mode 100644 crates/sqlbuild-cli/tests/native_commands/test_proxy.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_proxy.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json b/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json new file mode 100644 index 0000000000..2f75160f35 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json @@ -0,0 +1,9 @@ +{ + "origin": "https://control.example.com", + "person": "orders-reader", + "subject": "external-orders-reader", + "service_subject": "external-orders-service", + "absent_subject": "external-absent-reader", + "unknown_subject": "external-unbound-reader", + "untrusted_peer": "192.0.2.24" +} diff --git a/clients/typescript/README.md b/clients/typescript/README.md index 8bbaf3f47e..e0bd1b8117 100644 --- a/clients/typescript/README.md +++ b/clients/typescript/README.md @@ -38,3 +38,33 @@ refetching the authorized REST resources; the stream is not durable history. Browser `EventSource` cannot set a bearer header: use a fetch-based SSE consumer for the existing machine-authenticated endpoint. Human browser sessions are a separate authentication contract, not an inferred privilege of this client. + +## Authenticate through a trusted proxy + +Create a person with `sqb admin create-principal orders-reader`, then put the +returned `principal_id` in a server-owned JSON file. Subjects are exact ASCII +upstream identities, not inferred local usernames or email addresses: + +```json +{ + "origin": "https://control.example.com", + "trusted_peers": ["127.0.0.1"], + "identities": { + "external-orders-reader": "00000000-0000-0000-0000-000000000001" + } +} +``` + +Replace the example UUID with the created person's ID, then start +`sqb server --trusted-proxy-config proxy.json`. The proxy must terminate HTTPS, +remove any client-supplied `X-SQB-User`, and replace it with the authenticated +upstream subject. Restrict access to the HTTP backend to the configured proxy +peers. Trusting loopback also trusts other local processes on that host; +`Forwarded` and `X-Forwarded-For` never establish peer trust. + +Through that proxy, `GET /api/auth/proxy` returns the bound `principal_id` only. +It issues no cookie or bearer and grants no machine or resource authority. +Unbound subjects, missing or disabled people, and service accounts are denied; +accounts are not auto-provisioned. Password and trusted-proxy modes cannot be +combined. No default role or SSO group assignment is inferred by this identity +endpoint. diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 683f5af421..95f952fdc8 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -48,6 +48,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/auth/proxy": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** @description Requires a configured socket peer and an upstream-replaced X-SQB-User subject. Does not issue sessions or grant permissions. */ + get: operations["proxyIdentity"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/auth/session": { parameters: { query?: never; @@ -631,6 +648,10 @@ export interface components { }; /** @enum {string} */ PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; + ProxyIdentityResponse: { + /** Format: uuid */ + principal_id: string; + }; /** @description One invocation's records; the server owns its claim namespace, start time and version. */ RecallBatch: { invocation_id: string; @@ -1176,6 +1197,33 @@ export interface operations { }; }; }; + proxyIdentity: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProxyIdentityResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; currentSession: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 0c091e4e3e..fd65fcc8bd 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1051,6 +1051,13 @@ "help": "enable password sessions for this exact external HTTPS origin; requires HTTPS termination in front of the HTTP listener", "group": 1 }, + { + "kind": "store", + "options": ["--trusted-proxy-config"], + "dest": "control_trusted_proxy_config", + "help": "JSON origin, trusted_peers and explicit subject-to-person identities; proxy must replace X-SQB-User and terminate HTTPS; no provisioning or grants", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 8e78095dff..d24687a0b3 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use serde_json::json; +use sqlbuild_orchestration::api::models::ApiAuthentication; use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; @@ -253,6 +254,26 @@ pub(crate) fn control_address( Ok(Some(SocketAddr::new(host, port))) } +pub(crate) fn control_authentication( + namespace: &Namespace, +) -> Result { + let trusted_proxy = text(namespace, "control_trusted_proxy_config") + .map(|path| { + let document = std::fs::read_to_string(path).map_err(|_| { + ControlError::Invalid("could not read trusted-proxy configuration".into()) + })?; + serde_json::from_str(&document).map_err(|_| ControlError::Invalid( + "trusted-proxy configuration requires valid JSON with origin, trusted_peers and identities".into(), + )) + }) + .transpose()?; + Ok(ApiAuthentication { + password_origin: text(namespace, "control_password_origin"), + trusted_proxy, + ..ApiAuthentication::default() + }) +} + pub(crate) fn control_database(namespace: &Namespace) -> Result { let variable = text(namespace, "control_database_env") .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 4c737efd2d..a746732f10 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,7 +1,6 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::api::models::ApiAuthentication; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -9,7 +8,7 @@ use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, flag, text}; +use crate::native_commands::_helpers::decision::{control_address, control_authentication, flag}; use crate::native_commands::_helpers::execution::{ open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; @@ -28,6 +27,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; + let authentication = control_authentication(namespace)?; let store = open_control_store(namespace).await?; let workers = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); @@ -46,10 +46,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - ApiAuthentication { - password_origin: text(namespace, "control_password_origin"), - ..ApiAuthentication::default() - }, + authentication, stopped(shutdown.clone()), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index 1258cd6628..163c973a81 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -8,6 +8,8 @@ mod test_admin; mod test_control; #[path = "native_commands/admin/test_passwords.rs"] mod test_passwords; +#[path = "native_commands/test_proxy.rs"] +mod test_proxy; #[path = "native_commands/test_types.rs"] mod test_types; #[path = "native_commands/worker/test_worker.rs"] diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index 908b552555..df025a6a36 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -15,6 +15,10 @@ pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { pub(crate) fn start_control(arguments: &[&str]) -> TestControl { let directory = tempfile::tempdir().expect("isolated native control project"); + start_control_in(directory, arguments) +} + +pub(crate) fn start_control_in(directory: tempfile::TempDir, arguments: &[&str]) -> TestControl { let binary = isolated_binary(directory.path()); let stderr = directory.path().join("control.log"); let child = Command::new(binary) diff --git a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs new file mode 100644 index 0000000000..467f177080 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs @@ -0,0 +1,125 @@ +//! A standalone server loads explicit proxy bindings and rejects ambiguous authentication modes. + +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::helpers::{api_ready, start_control_in, wait_for_exit}; +use crate::test_types::{ProxyCliTestCase, ProxyConfigurationTestCase}; + +#[test] +fn given_an_owner_configured_proxy_when_a_standalone_server_authenticates_then_only_explicit_people_resolve_without_machine_authority() + { + let test_cases = [ProxyCliTestCase { + description: "explicit upstream subject differs from the local username", + subject: "external-orders-reader", + expected_status: StatusCode::OK, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let directory = tempfile::tempdir().expect("prepared proxy installation"); + let provisioned = std::process::Command::new(env!("CARGO_BIN_EXE_sqb")) + .args(["admin", "create-principal", "orders-reader"]) + .current_dir(directory.path()).env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL").output().expect("native owner provisioning"); + assert!(provisioned.status.success()); + let person: Value = serde_json::from_slice(&provisioned.stdout).expect("person JSON"); + let principal = person["principal_id"].as_str().expect("person UUID"); + let configuration = json!({"origin": "https://control.example.com", "trusted_peers": ["127.0.0.1"], "identities": {test_case.subject: principal}}); + std::fs::write(directory.path().join("proxy.json"), configuration.to_string()).expect("server-owned identity bindings"); + let mut control = start_control_in(directory, &["server", "--port", "0", "--no-scheduler", "--trusted-proxy-config", "proxy.json"]); + let url = api_ready(&mut control).await; + let client = Client::builder().timeout(std::time::Duration::from_secs(5)).build().expect("HTTP client"); + let response = client.get(format!("{url}/api/auth/proxy")) + .header(header::HOST, "control.example.com").header(header::ORIGIN, "https://control.example.com") + .header("x-sqb-user", test_case.subject).send().await.expect("upstream identity"); + assert_eq!(response.status(), test_case.expected_status, "{}", test_case.description); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + assert_eq!(response.json::().await.expect("identity"), json!({"principal_id": principal})); + let machine = client.get(format!("{url}/api/locations/{}/health", uuid::Uuid::now_v7())) + .header("x-sqb-user", test_case.subject).send().await.expect("machine boundary"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_invalid_or_conflicting_proxy_configuration_when_a_standalone_server_starts_then_it_fails_closed() + { + let test_cases = [ + ProxyConfigurationTestCase { + description: "malformed JSON", + configuration: "{", + arguments: &[], + expected_error: "trusted-proxy configuration requires valid JSON", + }, + ProxyConfigurationTestCase { + description: "unknown configuration fields", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{},"unknown":"synthetic-unrecognized-value"}"#, + arguments: &[], + expected_error: "trusted-proxy configuration requires valid JSON", + }, + ProxyConfigurationTestCase { + description: "no trusted peer boundary", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":[],"identities":{}}"#, + arguments: &[], + expected_error: "requires explicit trusted peer IP addresses", + }, + ProxyConfigurationTestCase { + description: "non-HTTPS external origin", + configuration: r#"{"origin":"http://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, + arguments: &[], + expected_error: "trusted-proxy authentication requires an HTTPS origin", + }, + ProxyConfigurationTestCase { + description: "subject cannot be represented by the header transport", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{"external-café":"00000000-0000-0000-0000-000000000001"}}"#, + arguments: &[], + expected_error: "subjects must be nonempty, unpadded ASCII header values", + }, + ProxyConfigurationTestCase { + description: "ambiguous password/proxy modes", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, + arguments: &["--password-origin", "https://control.example.com"], + expected_error: "trusted-proxy authentication cannot be combined", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let directory = tempfile::tempdir().expect("invalid proxy installation"); + std::fs::write(directory.path().join("proxy.json"), test_case.configuration) + .expect("configuration input"); + let mut arguments = vec![ + "server", + "--port", + "0", + "--no-scheduler", + "--trusted-proxy-config", + "proxy.json", + ]; + arguments.extend_from_slice(test_case.arguments); + let mut control = start_control_in(directory, &arguments); + let status = wait_for_exit(&mut control).await; + assert!(!status.success(), "{}", test_case.description); + let log = std::fs::read_to_string(&control.stderr).expect("startup failure"); + assert!( + log.contains(test_case.expected_error), + "{}", + test_case.description + ); + assert!( + !log.contains(test_case.configuration), + "configuration contents must not be echoed" + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 1d9cd298b0..73b0341411 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -41,6 +41,19 @@ pub(crate) struct PasswordOriginTestCase { pub(crate) expected_error: &'static str, } +pub(crate) struct ProxyCliTestCase { + pub(crate) description: &'static str, + pub(crate) subject: &'static str, + pub(crate) expected_status: reqwest::StatusCode, +} + +pub(crate) struct ProxyConfigurationTestCase { + pub(crate) description: &'static str, + pub(crate) configuration: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_error: &'static str, +} + pub(crate) struct TestControl { pub(crate) child: Child, pub(crate) directory: tempfile::TempDir, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs index 837ad3412c..043f8046c8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs @@ -4,7 +4,7 @@ use std::net::SocketAddr; use axum::Router; -use crate::api::_helpers::authentication::password_sessions::router; +use crate::api::_helpers::authentication::{password_sessions, trusted_proxy}; use crate::api::models::{ApiAuthentication, DevelopmentScope}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -17,6 +17,7 @@ pub(crate) fn configure( let ApiAuthentication { development_scope, password_origin, + trusted_proxy, } = authentication; if development_scope.is_some() && password_origin.is_some() { return Err(ControlError::Invalid( @@ -28,5 +29,14 @@ pub(crate) fn configure( "disabled authentication requires a loopback listener".into(), )); } - Ok((development_scope, router(store, password_origin)?)) + if trusted_proxy.is_some() && (development_scope.is_some() || password_origin.is_some()) { + return Err(ControlError::Invalid( + "trusted-proxy authentication cannot be combined with disabled or password authentication".into(), + )); + } + Ok(( + development_scope, + password_sessions::router(store, password_origin)? + .merge(trusted_proxy::router(store, trusted_proxy)?), + )) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs index 6684241c2a..eb3cf959d7 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod bearer; pub(crate) mod configure; pub(crate) mod password_sessions; pub(crate) mod password_transport; +pub(crate) mod trusted_proxy; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index d9db92f253..7856d363c2 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -6,14 +6,13 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router, middleware}; -use crate::api::_helpers::authentication::password_transport::{authenticate, deny, guard}; -use crate::api::constants::{ - HTTPS_SCHEME, PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE, ROOT_PATH, +use crate::api::_helpers::authentication::password_transport::{ + authenticate, deny, guard, parse_origin, }; +use crate::api::constants::{PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE}; use crate::api::errors::ApiError; use crate::api::models::{ - AuthenticationDenial, PasswordChangeRequest, PasswordLoginRequest, PasswordOrigin, - PasswordSessionResponse, + AuthenticationDenial, PasswordChangeRequest, PasswordLoginRequest, PasswordSessionResponse, }; use crate::credentials::constants::{LOGIN_WINDOW_SECONDS, PASSWORD_SESSION_SECONDS}; use crate::credentials::main::{ @@ -33,19 +32,7 @@ pub(crate) fn router( let Some(origin) = origin else { return Ok(Router::new()); }; - let url = reqwest::Url::parse(&origin).map_err(|_| { - ControlError::Invalid("password authentication requires an HTTPS origin".into()) - })?; - if url.scheme() != HTTPS_SCHEME - || url.host().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.path() != ROOT_PATH - || url.query().is_some() - || url.fragment().is_some() - { - return Err(ControlError::Invalid("password authentication requires an HTTPS origin without credentials, path, query or fragment".into())); - } + let origin = parse_origin(&origin, "password")?; let protected = Router::new() .route("/api/auth/session", get(current)) .route("/api/auth/logout", post(logout)) @@ -55,9 +42,7 @@ pub(crate) fn router( .route("/api/auth/password/login", post(login)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) .layer(middleware::from_fn_with_state(store.clone(), guard)) - .layer(Extension(PasswordOrigin( - url.origin().ascii_serialization(), - )))) + .layer(Extension(origin))) } #[utoipa::path(post, path = "/api/auth/password/login", operation_id = "loginPassword", diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs index 7f105fdec1..8d9d8d693c 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -10,17 +10,42 @@ use axum::{Extension, Json}; use serde_json::json; use uuid::Uuid; -use crate::api::constants::{PASSWORD_COOKIE, PASSWORD_CSRF_HEADER}; +use crate::api::constants::{HTTPS_SCHEME, PASSWORD_COOKIE, PASSWORD_CSRF_HEADER, ROOT_PATH}; use crate::api::errors::ApiError; -use crate::api::models::{AuthenticationDenial, PasswordOrigin}; +use crate::api::models::{AuthenticationDenial, BrowserOrigin}; use crate::credentials::main::authenticate_session::authenticate_session; use crate::credentials::main::record_authentication_denial::record_authentication_denial; use crate::credentials::models::{AuditContext, SessionAccess}; +use crate::store::errors::ControlError; use crate::store::models::ControlStore; +pub(crate) fn parse_origin(value: &str, mechanism: &str) -> Result { + let error = || { + ControlError::Invalid(format!( + "{mechanism} authentication requires an HTTPS origin without credentials, path, query or fragment" + )) + }; + let url = reqwest::Url::parse(value).map_err(|_| { + ControlError::Invalid(format!( + "{mechanism} authentication requires an HTTPS origin" + )) + })?; + if url.scheme() != HTTPS_SCHEME + || url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != ROOT_PATH + || url.query().is_some() + || url.fragment().is_some() + { + return Err(error()); + } + Ok(BrowserOrigin(url.origin().ascii_serialization())) +} + pub(crate) async fn guard( State(store): State, - Extension(origin): Extension, + Extension(origin): Extension, mut request: Request, next: Next, ) -> Result { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs new file mode 100644 index 0000000000..7f5f2191e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs @@ -0,0 +1,109 @@ +//! Proxy subjects resolve only through explicit server-owned bindings and the real socket peer. + +use std::sync::Arc; + +use axum::extract::State; +use axum::http::{HeaderMap, HeaderValue}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{Extension, Json, Router, middleware}; +use serde_json::json; + +use crate::api::_helpers::authentication::password_transport::{ + deny, guard, parse_origin, single_header, +}; +use crate::api::constants::PROXY_USER_HEADER; +use crate::api::errors::ApiError; +use crate::api::models::{AuthenticationDenial, ProxyIdentityResponse, TrustedProxyAuthentication}; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) fn router( + store: &ControlStore, + settings: Option, +) -> Result, ControlError> { + let Some(mut settings) = settings else { + return Ok(Router::new()); + }; + let origin = parse_origin(&settings.origin, "trusted-proxy")?; + if settings.trusted_peers.is_empty() { + return Err(ControlError::Invalid( + "trusted-proxy authentication requires explicit trusted peer IP addresses".into(), + )); + } + if settings.identities.keys().any(|subject| { + subject.trim().is_empty() + || subject.trim() != subject + || match HeaderValue::from_str(subject) { + Ok(value) => value.to_str().is_err(), + Err(_) => true, + } + }) { + return Err(ControlError::Invalid( + "trusted-proxy subjects must be nonempty, unpadded ASCII header values".into(), + )); + } + settings.trusted_peers = settings + .trusted_peers + .into_iter() + .map(|peer| peer.to_canonical()) + .collect(); + Ok(Router::new() + .route("/api/auth/proxy", get(identity)) + .layer(middleware::from_fn_with_state(store.clone(), guard)) + .layer(Extension(origin)) + .layer(Extension(Arc::new(settings)))) +} + +#[utoipa::path(get, path = "/api/auth/proxy", operation_id = "proxyIdentity", + description = "Requires a configured socket peer and an upstream-replaced X-SQB-User subject. Does not issue sessions or grant permissions.", + security(("proxy" = [])), + responses((status = 200, body = ProxyIdentityResponse), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn identity( + State(store): State, + Extension(settings): Extension>, + Extension(mut context): Extension, + headers: HeaderMap, +) -> Result { + if !context + .ip_address + .is_some_and(|peer| settings.trusted_peers.contains(&peer)) + { + return deny(&store, &context, "untrusted_proxy_peer").await; + } + let principal = single_header(&headers, PROXY_USER_HEADER) + .and_then(|subject| settings.identities.get(subject)) + .copied(); + let Some(principal_id) = principal else { + return deny(&store, &context, "unbound_proxy_subject").await; + }; + let transaction = begin(&store, TransactionMode::Write).await?; + let (transaction, rows) = query(transaction, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND kind='person' AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)]).await?; + if rows.is_empty() { + commit(transaction).await?; + return deny(&store, &context, "inactive_proxy_person").await; + } + context.actor = format!("principal:{principal_id}"); + context.granting_role = "trusted-proxy".into(); + let identity = ProxyIdentityResponse { principal_id }; + commit( + append_audit( + transaction, + &context, + (context.request_id, "proxy.authenticated"), + ( + json!({"principal_id": principal_id}), + json!(null), + json!(&identity), + ), + ) + .await?, + ) + .await?; + Ok(Json(identity).into_response()) +} diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index 7bb3346324..1ee030772c 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -6,5 +6,6 @@ pub(crate) const PASSWORD_BODY_BYTES: usize = 16 * 1024; pub(crate) const PASSWORD_COOKIE: &str = "__Host-sqb-session"; pub(crate) const PASSWORD_CSRF_COOKIE: &str = "__Host-sqb-csrf"; pub(crate) const PASSWORD_CSRF_HEADER: &str = "x-sqb-csrf"; +pub(crate) const PROXY_USER_HEADER: &str = "x-sqb-user"; pub(crate) const HTTPS_SCHEME: &str = "https"; pub(crate) const ROOT_PATH: &str = "/"; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs index 707449538e..e7a46a5b8e 100644 --- a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -3,7 +3,7 @@ use utoipa::OpenApi; use utoipa::openapi::security::{ApiKey, ApiKeyValue, HttpAuthScheme, HttpBuilder, SecurityScheme}; -use crate::api::constants::PASSWORD_COOKIE; +use crate::api::constants::{PASSWORD_COOKIE, PROXY_USER_HEADER}; use crate::api::models::ControlApi; #[must_use] @@ -24,4 +24,14 @@ pub fn openapi_document() -> utoipa::openapi::OpenApi { SecurityScheme::ApiKey(ApiKey::Cookie(ApiKeyValue::new(PASSWORD_COOKIE))), ); document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "proxy", + SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::with_description( + PROXY_USER_HEADER, + "Upstream-replaced subject; only accepted from explicitly configured socket peers", + ))), + ); + document } diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c03eace64a..bebb162ccb 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::api::_helpers::authentication::password_sessions; +use crate::api::_helpers::authentication::trusted_proxy; use crate::api::_helpers::resources::{ changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, @@ -14,6 +15,7 @@ use crate::api::_helpers::resources::{ password_sessions::current, password_sessions::logout, password_sessions::change_password, + trusted_proxy::identity, changes::subscribe, location_health::report, location_health::recent, @@ -100,10 +102,25 @@ pub struct DevelopmentScope { pub struct ApiAuthentication { pub development_scope: Option, pub password_origin: Option, + pub trusted_proxy: Option, } #[derive(Clone, Debug)] -pub(crate) struct PasswordOrigin(pub(crate) String); +pub(crate) struct BrowserOrigin(pub(crate) String); + +/// Explicit person bindings require an upstream-replaced X-SQB-User; no provisioning or grants. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TrustedProxyAuthentication { + pub origin: String, + pub trusted_peers: std::collections::BTreeSet, + pub identities: std::collections::BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProxyIdentityResponse { + pub principal_id: uuid::Uuid, +} #[derive(Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index d69d358ad5..1cac36cbb2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -20,6 +20,8 @@ mod test_location_health; mod test_location_work; #[path = "test_passwords.rs"] mod test_passwords; +#[path = "test_proxy.rs"] +mod test_proxy; #[path = "test_reads.rs"] mod test_reads; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_proxy.rs b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs new file mode 100644 index 0000000000..fb04184725 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs @@ -0,0 +1,261 @@ +//! Explicit proxy bindings authenticate known people without provisioning or machine authority. + +use std::collections::{BTreeMap, BTreeSet}; + +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, AuthenticationDenial, TrustedProxyAuthentication}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::ProxyApiTestCase; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_explicit_proxy_bindings_when_real_peers_and_live_people_are_checked_then_spoofing_and_implicit_grants_are_rejected() + { + let test_cases = [ + ProxyApiTestCase { + description: "SQLite proxy identity boundary", + backend: Backend::Sqlite, + trusted_peer: "::ffff:127.0.0.1", + expected_authentications: 2, + }, + ProxyApiTestCase { + description: "PostgreSQL proxy identity boundary", + backend: Backend::Postgres, + trusted_peer: "127.0.0.1", + expected_authentications: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json"); + let inputs: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop subjects")) + .expect("synthetic bindings"); + let value = |key: &str| inputs[key].as_str().expect("text input"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = + create_principal(store, &context, (value("person"), PrincipalKind::Person)) + .await + .expect("known person"); + let service = create_principal( + store, + &context, + ("orders-service", PrincipalKind::ServiceAccount), + ) + .await + .expect("service account"); + let settings = TrustedProxyAuthentication { + origin: value("origin").into(), + trusted_peers: BTreeSet::from([test_case.trusted_peer.parse().expect("proxy IP")]), + identities: BTreeMap::from([ + (value("subject").into(), person), + (value("service_subject").into(), service), + (value("absent_subject").into(), Uuid::now_v7()), + ]), + }; + let api = start_api_with_authentication( + store, + ApiAuthentication { + trusted_proxy: Some(settings.clone()), + ..ApiAuthentication::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + trusted_proxy: Some(settings.clone()), + ..ApiAuthentication::default() + }, + ) + .await; + let untrusted = start_api_with_authentication( + store, + ApiAuthentication { + trusted_proxy: Some(TrustedProxyAuthentication { + trusted_peers: BTreeSet::from([value("untrusted_peer") + .parse() + .expect("untrusted IP")]), + ..settings + }), + ..ApiAuthentication::default() + }, + ) + .await; + let client = Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let request = |url: &str| { + client + .get(format!("{url}/api/auth/proxy")) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, value("origin")) + }; + for url in [&api.url, &other.url] { + let response = request(url) + .header("x-sqb-user", value("subject")) + .send() + .await + .expect("proxy authentication"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + assert_eq!( + response.json::().await.expect("identity only"), + json!({"principal_id": person}) + ); + } + let mut denials = Vec::new(); + for subject in [ + "", + value("unknown_subject"), + value("person"), + value("service_subject"), + value("absent_subject"), + ] { + let response = request(&api.url) + .header("x-sqb-user", subject) + .send() + .await + .expect("unbound/ineligible subject"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + denials.push( + response + .json::() + .await + .expect("durable denial") + .decision_id, + ); + } + for probe in [ + request(&api.url), + request(&api.url) + .header("x-sqb-user", value("subject")) + .header("x-sqb-user", value("subject")), + request(&api.url) + .header("x-sqb-user", value("subject")) + .headers(header::HeaderMap::from_iter([( + header::ORIGIN, + header::HeaderValue::from_static("https://other.example.com"), + )])), + request(&api.url) + .header("x-sqb-user", value("subject")) + .headers(header::HeaderMap::from_iter([( + header::HOST, + header::HeaderValue::from_static("other.example.com"), + )])), + request(&untrusted.url) + .header("x-sqb-user", value("subject")) + .header("x-forwarded-for", value("untrusted_peer")) + .header("forwarded", format!("for={}", value("untrusted_peer"))), + ] { + let response = probe.send().await.expect("spoofing probe"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + denials.push( + response + .json::() + .await + .expect("spoofing denial") + .decision_id, + ); + } + let machine = client + .get(format!( + "{}/api/locations/{}/health", + api.url, + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header("x-sqb-user", value("subject")) + .send() + .await + .expect("no location authority"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("disable person"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(now), Parameter::Id(person)], + ) + .await + .expect("live disabled fact"); + commit(transaction).await.expect("disable committed"); + for url in [&api.url, &other.url] { + let response = request(url) + .header("x-sqb-user", value("subject")) + .send() + .await + .expect("live person validation"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + denials.push( + response + .json::() + .await + .expect("disabled denial") + .decision_id, + ); + } + let mut transaction = begin(store, TransactionMode::Read) + .await + .expect("inspect durable evidence"); + for decision in denials { + let rows; + (transaction, rows) = query(transaction, "SELECT actor, ip_address, request_id FROM sqb_security_audit WHERE event_id=$1 AND action='authentication.denied'", vec![Parameter::Id(decision)]).await.expect("denial audit"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["actor"], "anonymous"); + assert_eq!(rows[0]["ip_address"], "127.0.0.1"); + assert_eq!(rows[0]["request_id"], json!(decision)); + } + let (transaction, rows) = query(transaction, "SELECT actor, granting_role, ip_address FROM sqb_security_audit WHERE action='proxy.authenticated'", vec![]).await.expect("proxy audit"); + assert_eq!( + rows.len(), + test_case.expected_authentications, + "{}", + test_case.description + ); + for row in rows { + assert_eq!(row["actor"], format!("principal:{person}")); + assert_eq!(row["granting_role"], "trusted-proxy"); + assert_eq!(row["ip_address"], "127.0.0.1"); + } + let (transaction, rows) = query(transaction, "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_password_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens", vec![]).await.expect("no implicit provisioning"); + assert_eq!( + json!(rows[0]), + json!({"principals": 2, "sessions": 0, "tokens": 0}) + ); + commit(transaction).await.expect("inspection complete"); + stop_api(untrusted).await; + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 2efde86eea..2a74905058 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -80,3 +80,10 @@ pub(crate) struct PasswordApiTestCase { pub(crate) backend: Backend, pub(crate) expected_lifetime: i64, } + +pub(crate) struct ProxyApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) trusted_peer: &'static str, + pub(crate) expected_authentications: usize, +} From b1b3ec57bed24de93d2c58daee9e621e191846c7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:24:37 +0000 Subject: [PATCH 074/138] test: preserve worker execution coverage under derived-relation rules --- .../orchestration/rerun_from_failure.sql | 3 ++- .../publication/test_docker_worker.py | 4 ++-- .../compiled_project/publication/test_worker.py | 14 +++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql index 8304e49843..79baa81e01 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql +++ b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql @@ -1,8 +1,9 @@ MODEL (description 'Retry a failed cast after warehouse input changes.', materialized table); +WITH source(value) AS (VALUES ('invalid')) SELECT CASE WHEN EXISTS ( SELECT 1 FROM information_schema.tables WHERE table_name = 'retry_ready' ) THEN 17 ELSE CAST(value AS INTEGER) END AS amount -FROM (VALUES ('invalid')) AS source(value) +FROM source diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index 05a6f35e07..c858549b22 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -43,7 +43,7 @@ [ WorkerExecutionTestCase( "pinned container inherits project context but not privileged credentials", - "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('37')) t(value)", + "WITH source(value) AS (VALUES ('37')) SELECT CAST(value AS BIGINT) AS amount FROM source", "SELECT amount FROM orders", "success", [(37,)], @@ -65,7 +65,7 @@ ), WorkerExecutionTestCase( "container warehouse error retains failed node evidence", - "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS BIGINT) AS amount FROM source", "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", "failed", [(0,)], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 391f828b68..7e2793f754 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -49,7 +49,7 @@ [ GroupExecutionTestCase( "group execution preserves independent downstream work", - "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source", "SELECT amount FROM orders", "partial", [(11,)], @@ -87,7 +87,7 @@ ), GroupExecutionTestCase( "whole downstream job is blocked without building an empty selection", - "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source", "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", "partial", [(0,)], @@ -96,7 +96,7 @@ ), GroupExecutionTestCase( "job failure stops the group", - "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source", "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", "failed", [(0,)], @@ -107,7 +107,7 @@ ), GroupExecutionTestCase( "critical node stops the group under the default policy", - "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source", "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", "failed", [(0,)], @@ -117,7 +117,7 @@ files={ "models/broken_orders.sql": ( "MODEL (description 'Critical orders.', materialized table, critical true);\n" - "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)\n" + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source\n" ), }, ), @@ -398,7 +398,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ [ WorkerExecutionTestCase( "successful model writes actual rows", - "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('23')) t(value)", + "WITH source(value) AS (VALUES ('23')) SELECT CAST(value AS BIGINT) AS amount FROM source", "SELECT amount FROM orders", "success", [(23,)], @@ -443,7 +443,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ ), WorkerExecutionTestCase( "warehouse error retains failed node evidence", - "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS BIGINT) AS amount FROM source", "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", "failed", [(0,)], From 0b28f8286b7405efbcd6005812f96f9c3a3af668 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:24:45 +0000 Subject: [PATCH 075/138] feat: verify explicitly bound OIDC identities with browser-bound PKCE --- Cargo.lock | 1292 ++++++++++++++++- Cargo.toml | 3 + .../orchestration/oidc_identities.json | 15 + clients/typescript/README.md | 42 + clients/typescript/src/schema.d.ts | 104 +- .../sqlbuild-cli/src/argv/command_table.json | 7 + .../src/native_commands/_helpers/decision.rs | 36 +- .../src/native_commands/_helpers/execution.rs | 18 +- .../src/native_commands/main/_run_control.rs | 15 +- .../tests/native_commands/test_proxy.rs | 88 +- .../tests/native_commands/test_types.rs | 3 +- crates/sqlbuild-orchestration/Cargo.toml | 3 + .../api/_helpers/authentication/configure.rs | 15 +- .../src/api/_helpers/authentication/mod.rs | 2 + .../src/api/_helpers/authentication/oidc.rs | 155 ++ .../_helpers/authentication/oidc_provider.rs | 183 +++ .../authentication/password_sessions.rs | 39 +- .../authentication/password_transport.rs | 24 +- .../_helpers/authentication/trusted_proxy.rs | 61 +- .../src/api/constants.rs | 2 + .../src/api/main/serve_api.rs | 11 +- crates/sqlbuild-orchestration/src/api/mod.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 32 +- .../sqlbuild-orchestration/src/api/types.rs | 12 + .../src/credentials/constants.rs | 1 + .../main/authenticate_bound_person.rs | 47 + .../src/credentials/main/claim_oidc_flow.rs | 66 + .../src/credentials/main/create_oidc_flow.rs | 47 + .../src/credentials/main/login_password.rs | 7 +- .../src/credentials/main/mod.rs | 5 +- ...t.rs => reserve_authentication_attempt.rs} | 23 +- .../src/credentials/models.rs | 25 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 7 +- .../src/store/oidc_login_flows.sql | 10 + .../tests/api/helpers.rs | 11 +- .../tests/api/oidc/helpers.rs | 259 ++++ .../tests/api/oidc/test_types.rs | 39 + .../tests/api/test_contracts.rs | 6 + .../tests/api/test_location_work.rs | 2 +- .../tests/api/test_oidc.rs | 352 +++++ .../tests/api/test_types.rs | 12 + .../tests/api/warehouse_metadata/helpers.rs | 3 +- .../credentials/sessions/test_contract.rs | 9 +- .../tests/sensors/test_runs.rs | 1 + .../tests/snapshots/helpers.rs | 2 +- 46 files changed, 2889 insertions(+), 209 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs create mode 100644 crates/sqlbuild-orchestration/src/api/types.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs rename crates/sqlbuild-orchestration/src/credentials/main/{reserve_password_attempt.rs => reserve_authentication_attempt.rs} (78%) create mode 100644 crates/sqlbuild-orchestration/src/store/oidc_login_flows.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_oidc.rs diff --git a/Cargo.lock b/Cargo.lock index 97e9dbad01..228f904fae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,15 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -65,6 +74,45 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "atoi" version = "2.0.0" @@ -166,6 +214,34 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-server" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" +dependencies = [ + "arc-swap", + "bytes", + "either", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.22.1" @@ -190,7 +266,16 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" dependencies = [ - "bit-vec", + "bit-vec 0.10.1", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", ] [[package]] @@ -203,6 +288,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" @@ -215,7 +306,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -241,6 +332,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -274,6 +374,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" version = "1.13.1" @@ -369,6 +478,12 @@ dependencies = [ "phf 0.12.1", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "collection_literals" version = "1.0.3" @@ -409,6 +524,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -427,6 +554,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -514,6 +647,22 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crypto-bigint" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -524,6 +673,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + [[package]] name = "ctrlc" version = "3.5.2" @@ -535,14 +695,61 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ctutils" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a" +dependencies = [ + "cmov", + "subtle", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -559,17 +766,123 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468 1.0.0", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + [[package]] name = "derive-where" version = "1.7.0" @@ -596,7 +909,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.119", @@ -618,18 +931,31 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dispatch2" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.13.1", "block2", "libc", "objc2", @@ -652,12 +978,79 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" +dependencies = [ + "der 0.8.3", + "digest 0.11.3", + "elliptic-curve", + "rfc6979", + "signature 3.0.0", + "spki 0.8.1", + "zeroize", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "signature 3.0.0", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.11.1", + "signature 3.0.0", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "elliptic-curve" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" +dependencies = [ + "base16ct", + "crypto-bigint", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff", + "group", + "hybrid-array", + "pem-rfc7468 1.0.0", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -723,9 +1116,25 @@ dependencies = [ "globset", "serde", "serde_json", - "sha2", + "sha2 0.10.9", ] +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "find-msvc-tools" version = "0.1.11" @@ -764,6 +1173,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -920,6 +1339,42 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff", + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" @@ -964,7 +1419,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -973,7 +1428,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -1030,6 +1494,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "subtle", + "typenum", + "zeroize", +] + [[package]] name = "hyper" version = "1.12.0" @@ -1038,7 +1513,9 @@ checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", + "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1224,6 +1701,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -1269,6 +1757,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.15.0" @@ -1284,6 +1781,60 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "js-sys" version = "0.3.107" @@ -1305,12 +1856,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libmimalloc-sys" version = "0.1.49" @@ -1326,7 +1892,7 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ - "bitflags", + "bitflags 2.13.1", "libc", "plain", "redox_syscall 0.9.5", @@ -1412,7 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -1445,6 +2011,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -1472,7 +2044,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags", + "bitflags 2.13.1", "cfg-if", "libc", ] @@ -1483,12 +2055,73 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1496,6 +2129,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", +] + +[[package]] +name = "oauth2" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" +dependencies = [ + "base64 0.22.1", + "chrono", + "getrandom 0.2.17", + "http", + "rand 0.8.8", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "sha2 0.10.9", + "thiserror 1.0.69", + "url", ] [[package]] @@ -1522,19 +2176,96 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openidconnect" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0d098a59353679624d406825940139e2b2d554a52b64c9fb1b1bc42a11fe5c" +dependencies = [ + "base64 0.22.1", + "chrono", + "dyn-clone", + "ed25519-dalek", + "getrandom 0.4.3", + "hmac 0.12.1", + "http", + "itertools 0.14.0", + "log", + "oauth2", + "p256", + "p384", + "rand 0.8.8", + "rsa", + "serde", + "serde-value", + "serde_json", + "serde_path_to_error", + "serde_plain", + "serde_with", + "sha2 0.10.9", + "subtle", + "thiserror 1.0.69", + "url", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordermap" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" dependencies = [ - "indexmap", + "indexmap 2.14.2", +] + +[[package]] +name = "p256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primefield", + "primeorder", + "sha2 0.11.1", +] + +[[package]] +name = "p384" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" +dependencies = [ + "ecdsa", + "elliptic-curve", + "fiat-crypto", + "primefield", + "primeorder", + "sha2 0.11.1", ] [[package]] @@ -1554,27 +2285,55 @@ dependencies = [ ] [[package]] -name = "parking_lot_core" -version = "0.9.12" +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.5.18", - "smallvec", - "windows-link", + "base64ct", ] [[package]] -name = "password-hash" -version = "0.5.0" +name = "pem-rfc7468" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" dependencies = [ "base64ct", - "rand_core 0.6.4", - "subtle", ] [[package]] @@ -1645,6 +2404,37 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.3", + "spki 0.8.1", +] + [[package]] name = "pkg-config" version = "0.3.34" @@ -1693,6 +2483,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -1702,6 +2501,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1711,6 +2516,33 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common 0.2.2", + "ff", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -1747,8 +2579,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" dependencies = [ "bit-set", - "bit-vec", - "bitflags", + "bit-vec 0.10.1", + "bitflags 2.13.1", "chacha20", "core_detect", "num-traits", @@ -2021,13 +2853,27 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.1", ] [[package]] @@ -2036,7 +2882,27 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" dependencies = [ - "bitflags", + "bitflags 2.13.1", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] @@ -2106,6 +2972,16 @@ dependencies = [ "webpki-roots 1.0.11", ] +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint", + "hmac 0.13.0", +] + [[package]] name = "ring" version = "0.17.14" @@ -2131,6 +3007,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + [[package]] name = "rtoolbox" version = "0.0.6" @@ -2149,7 +3045,7 @@ checksum = "a12d40253033e38020db1bdb91a2cf5d0bee1e755c1938a26d43fa3c73d2b426" dependencies = [ "aho-corasick", "arrayvec", - "bitflags", + "bitflags 2.13.1", "compact_str", "get-size2", "is-macro", @@ -2168,7 +3064,7 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "154345bd168f768a36f620180eabc9a5bae844d8c853182b1a24bb1a984c9b00" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bstr", "compact_str", "get-size2", @@ -2190,7 +3086,7 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dc22c8e07f58267a6c517ab87df57f503ef92927cd0fc46c8445ff318a39e54" dependencies = [ - "itertools", + "itertools 0.15.0", "ruff_source_file", "ruff_text_size", "rustc-hash", @@ -2222,7 +3118,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" dependencies = [ - "bitflags", + "bitflags 2.13.1", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -2236,13 +3132,31 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -2327,12 +3241,50 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct", + "ctutils", + "der 0.8.3", + "hybrid-array", + "subtle", + "zeroize", +] + [[package]] name = "semver" version = "1.0.28" @@ -2349,6 +3301,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -2375,7 +3337,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -2394,6 +3356,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -2415,6 +3386,49 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -2429,7 +3443,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -2458,6 +3483,26 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -2495,6 +3540,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.3", +] + [[package]] name = "sqlbuild-analysis" version = "0.186.2" @@ -2507,7 +3578,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-attachments", "sqlbuild-cache", "sqlbuild-core", @@ -2551,7 +3622,7 @@ dependencies = [ "blake3", "rayon", "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-core", "tempfile", "walkdir", @@ -2602,7 +3673,7 @@ name = "sqlbuild-core" version = "0.186.2" dependencies = [ "regex", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -2647,13 +3718,16 @@ version = "0.186.2" dependencies = [ "argon2", "axum", + "axum-server", "chrono", "chrono-tz", "croner", "flate2", "futures-util", "getrandom 0.4.3", + "openidconnect", "proptest", + "rcgen", "reqwest", "rusqlite", "semver", @@ -2687,7 +3761,7 @@ dependencies = [ "ruff_python_ast", "ruff_python_parser", "ruff_text_size", - "sha2", + "sha2 0.10.9", "sqlbuild-core", ] @@ -2705,7 +3779,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", @@ -2743,7 +3817,7 @@ dependencies = [ "rustix", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-config", "sqlbuild-core", "tempfile", @@ -2789,7 +3863,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-analysis", "sqlbuild-attachments", "sqlbuild-cache", @@ -2807,7 +3881,7 @@ name = "sqlbuild-scopes" version = "0.186.2" dependencies = [ "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", @@ -2829,7 +3903,7 @@ name = "sqlbuild-state" version = "0.186.2" dependencies = [ "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-core", "sqlbuild-cursor", ] @@ -2845,7 +3919,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -2866,7 +3940,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap", + "indexmap 2.14.2", "log", "memchr", "once_cell", @@ -2892,7 +3966,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", - "bitflags", + "bitflags 2.13.1", "byteorder", "chrono", "crc", @@ -2903,7 +3977,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -2913,7 +3987,7 @@ dependencies = [ "rand 0.8.8", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -3023,6 +4097,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -3099,6 +4184,36 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.5" @@ -3163,13 +4278,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap", + "indexmap 2.14.2", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -3202,7 +4331,7 @@ version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ - "indexmap", + "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -3244,7 +4373,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -3401,6 +4530,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3424,7 +4554,7 @@ version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" dependencies = [ - "indexmap", + "indexmap 2.14.2", "serde", "serde_json", "utoipa-gen", @@ -3878,12 +5008,52 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff", + "group", + "hybrid-array", + "primefield", +] + [[package]] name = "writeable" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.4" @@ -3904,7 +5074,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.3", - "synstructure", + "synstructure 0.14.0", ] [[package]] @@ -3945,7 +5115,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.3", - "synstructure", + "synstructure 0.14.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 567c8c800f..652b497017 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ rust-version = "1.88" [workspace.dependencies] argon2 = "0.5.3" axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } +axum-server = { version = "0.8", default-features = false, features = ["tls-rustls-no-provider"] } fensu-policy = "=0.22.0" futures-util = { version = "0.3", default-features = false, features = ["std"] } # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. @@ -78,6 +79,7 @@ mimalloc = { version = "0.1", default-features = false, features = ["local_dynam proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } rpassword = "7.5.4" +rcgen = "0.14" polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", @@ -96,6 +98,7 @@ bstr = "1" either = { version = "1", default-features = false, features = ["use_std"] } memchr = "2" once_cell = "1" +openidconnect = { version = "4.1", default-features = false, features = ["reqwest", "rustls-tls"] } regex-automata = { version = "0.4", default-features = false, features = ["dfa-search"] } syn-v2-macro-features = { package = "syn", version = "2", default-features = false, features = ["extra-traits", "fold", "visit-mut"] } syn-v3-macro-features = { package = "syn", version = "3", default-features = false, features = ["extra-traits", "full"] } diff --git a/benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json b/benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json new file mode 100644 index 0000000000..22a5eb3f2c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json @@ -0,0 +1,15 @@ +{ + "origin": "https://control.example.com", + "subject": "orders-person", + "invalid_tokens": [ + {"description": "issuer", "claims": {"iss": "https://other.example.com"}}, + {"description": "audience", "claims": {"aud": ["customers-control"]}}, + {"description": "nonce", "claims": {"nonce": "other-nonce"}}, + {"description": "expired", "claims": {"exp": 1}}, + {"description": "signature", "signing_key": "customers"}, + {"description": "access hash", "response": {"access_token": "customers-access"}}, + {"description": "missing token", "remove_fields": ["id_token"]} + ], + "ineligible_subjects": ["unknown-person", "orders-service", "absent-person"], + "lifecycle": ["login", "browser-mismatch", "replica-callback", "replay", "disabled-person", "expired-flow"] +} diff --git a/clients/typescript/README.md b/clients/typescript/README.md index e0bd1b8117..153eae0664 100644 --- a/clients/typescript/README.md +++ b/clients/typescript/README.md @@ -68,3 +68,45 @@ Unbound subjects, missing or disabled people, and service accounts are denied; accounts are not auto-provisioned. Password and trusted-proxy modes cannot be combined. No default role or SSO group assignment is inferred by this identity endpoint. + +## Verify an OIDC identity + +Register an authorization-code client with your HTTPS identity provider, using +`https://control.example.com/api/auth/oidc/callback` as its redirect URI. Create +each person through the local control-store-owner administration path and put +their UUID and exact provider subject in a server-owned JSON configuration: + +```json +{ + "origin": "https://control.example.com", + "issuer": "https://identity.example.com", + "client_id": "orders-control", + "identities": { + "external-orders-reader": "00000000-0000-0000-0000-000000000001" + } +} +``` + +Start `sqb server --oidc-config oidc.json` behind an HTTPS-terminating proxy that +preserves the configured Host and restricts access to the HTTP backend. For a +confidential client, add `client_secret_env` naming the server environment +variable holding its secret; never put the secret in the JSON file. To trust a +private provider CA, add `certificate_authority` pointing to a PEM bundle. +Relative certificate paths resolve from the server process's working directory. +TLS verification remains mandatory and upstream HTTP redirects are not followed. +Password, proxy, disabled-auth and OIDC human-authentication modes are exclusive. + +Navigate the browser to `/api/auth/oidc/login`. The server redirects to the +provider using S256 PKCE and sets a Secure, HttpOnly, host-only, five-minute +SameSite=Lax flow cookie. The provider's GET callback must return through that +same browser. State is single-use across server replicas; a changed server +identity configuration requires starting a fresh flow. The server validates +the ID token's signature, issuer, audience, expiry and nonce, and an access-token +hash when supplied. Provider keys are refreshed for each exchange. + +The callback returns only `principal_id` and clears the flow cookie. Unbound, +missing or disabled people and service accounts are denied. This identity-only +endpoint does not create an account, browser/CLI session, bearer or role grant, +and cannot authenticate machine routes. It is not a complete browser SSO or +`sqb login` workflow. Authentication denial and successful identity verification +are audited; flow starts share a 20-per-minute real-peer limit across replicas. diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 95f952fdc8..6036987cce 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -16,6 +16,40 @@ export interface paths { patch?: never; trace?: never; }; + "/api/auth/oidc/callback": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** @description Validates one browser-bound code and returns the explicitly bound person only; no session, bearer, provisioning or grants. */ + get: operations["oidcIdentity"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/oidc/login": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** @description Begins a browser-bound code flow with S256 PKCE; no account provisioning or grants. */ + get: operations["loginOidc"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/auth/password": { parameters: { query?: never; @@ -459,6 +493,10 @@ export interface components { /** @enum {string} */ status: "timed_out"; }; + ExternalIdentityResponse: { + /** Format: uuid */ + principal_id: string; + }; /** @description Fingerprint observations from one registered reader, independent of automatic polling. */ FingerprintBatch: { /** Format: int64 */ @@ -648,10 +686,6 @@ export interface components { }; /** @enum {string} */ PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; - ProxyIdentityResponse: { - /** Format: uuid */ - principal_id: string; - }; /** @description One invocation's records; the server owns its claim namespace, start time and version. */ RecallBatch: { invocation_id: string; @@ -1126,6 +1160,66 @@ export interface operations { }; }; }; + oidcIdentity: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ExternalIdentityResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + loginOidc: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 303: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; changePassword: { parameters: { query?: never; @@ -1211,7 +1305,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ProxyIdentityResponse"]; + "application/json": components["schemas"]["ExternalIdentityResponse"]; }; }; 401: { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index fd65fcc8bd..ad96be4dc1 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1058,6 +1058,13 @@ "help": "JSON origin, trusted_peers and explicit subject-to-person identities; proxy must replace X-SQB-User and terminate HTTPS; no provisioning or grants", "group": 1 }, + { + "kind": "store", + "options": ["--oidc-config"], + "dest": "control_oidc_config", + "help": "JSON HTTPS origin/issuer, client_id and explicit subject-to-person identities; native code flow with PKCE; no provisioning or grants", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index d24687a0b3..e27f177e38 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -257,23 +257,37 @@ pub(crate) fn control_address( pub(crate) fn control_authentication( namespace: &Namespace, ) -> Result { - let trusted_proxy = text(namespace, "control_trusted_proxy_config") - .map(|path| { - let document = std::fs::read_to_string(path).map_err(|_| { - ControlError::Invalid("could not read trusted-proxy configuration".into()) - })?; - serde_json::from_str(&document).map_err(|_| ControlError::Invalid( - "trusted-proxy configuration requires valid JSON with origin, trusted_peers and identities".into(), - )) - }) - .transpose()?; Ok(ApiAuthentication { password_origin: text(namespace, "control_password_origin"), - trusted_proxy, + trusted_proxy: authentication_settings( + text(namespace, "control_trusted_proxy_config"), + ("trusted-proxy", "origin, trusted_peers and identities"), + )?, + oidc: authentication_settings( + text(namespace, "control_oidc_config"), + ("OIDC", "origin, issuer, client_id and identities"), + )?, ..ApiAuthentication::default() }) } +fn authentication_settings( + path: Option, + (mechanism, fields): (&str, &str), +) -> Result, ControlError> { + path.map(|path| { + let document = std::fs::read_to_string(path).map_err(|_| { + ControlError::Invalid(format!("could not read {mechanism} configuration")) + })?; + serde_json::from_str(&document).map_err(|_| { + ControlError::Invalid(format!( + "{mechanism} configuration requires valid JSON with {fields}" + )) + }) + }) + .transpose() +} + pub(crate) fn control_database(namespace: &Namespace) -> Result { let variable = text(namespace, "control_database_env") .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 939cddd409..9859011f57 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -90,7 +90,12 @@ pub(crate) async fn run_development_roles( development_scope: Some(scope), ..ApiAuthentication::default() }, - development_stopped(shutdown.clone()), + (move || { + drop(write_output(&[(Stream::Stderr, &format!( + "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", + if triggers { "follow code defaults and operator overrides" } else { "start paused" } + ))])); + }, development_stopped(shutdown.clone())), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; tasks.spawn(development_location_role( @@ -119,17 +124,6 @@ pub(crate) async fn run_development_roles( "Development scheduler started\n", )])); } - drop(write_output(&[( - Stream::Stderr, - &format!( - "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", - if triggers { - "follow code defaults and operator overrides" - } else { - "start paused" - } - ), - )])); supervise_control_roles(tasks, stop).await } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index a746732f10..0be6102190 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -39,15 +39,20 @@ pub(crate) async fn run_control( "Starting control API...\n", )])); let listener = tokio::net::TcpListener::bind(address).await?; - drop(write_output(&[( - Stream::Stderr, - &format!("Control API ready: {}\n", listener.local_addr()?), - )])); + let address = listener.local_addr()?; tasks.spawn(serve_api( store.clone(), listener, authentication, - stopped(shutdown.clone()), + ( + move || { + drop(write_output(&[( + Stream::Stderr, + &format!("Control API ready: {address}\n"), + )])); + }, + stopped(shutdown.clone()), + ), )); start_platform_roles(&mut tasks, (&store, &shutdown))?; } diff --git a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs index 467f177080..1afdbe36a0 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs @@ -4,7 +4,7 @@ use reqwest::{Client, StatusCode, header}; use serde_json::{Value, json}; use crate::helpers::{api_ready, start_control_in, wait_for_exit}; -use crate::test_types::{ProxyCliTestCase, ProxyConfigurationTestCase}; +use crate::test_types::{AuthenticationConfigurationTestCase, ProxyCliTestCase}; #[test] fn given_an_owner_configured_proxy_when_a_standalone_server_authenticates_then_only_explicit_people_resolve_without_machine_authority() @@ -49,45 +49,100 @@ fn given_an_owner_configured_proxy_when_a_standalone_server_authenticates_then_o } #[test] -fn given_invalid_or_conflicting_proxy_configuration_when_a_standalone_server_starts_then_it_fails_closed() +fn given_invalid_or_conflicting_identity_configuration_when_a_standalone_server_starts_then_it_fails_closed() { let test_cases = [ - ProxyConfigurationTestCase { + AuthenticationConfigurationTestCase { description: "malformed JSON", + option: "--trusted-proxy-config", configuration: "{", arguments: &[], expected_error: "trusted-proxy configuration requires valid JSON", }, - ProxyConfigurationTestCase { + AuthenticationConfigurationTestCase { description: "unknown configuration fields", + option: "--trusted-proxy-config", configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{},"unknown":"synthetic-unrecognized-value"}"#, arguments: &[], expected_error: "trusted-proxy configuration requires valid JSON", }, - ProxyConfigurationTestCase { + AuthenticationConfigurationTestCase { description: "no trusted peer boundary", + option: "--trusted-proxy-config", configuration: r#"{"origin":"https://control.example.com","trusted_peers":[],"identities":{}}"#, arguments: &[], expected_error: "requires explicit trusted peer IP addresses", }, - ProxyConfigurationTestCase { + AuthenticationConfigurationTestCase { description: "non-HTTPS external origin", + option: "--trusted-proxy-config", configuration: r#"{"origin":"http://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, arguments: &[], expected_error: "trusted-proxy authentication requires an HTTPS origin", }, - ProxyConfigurationTestCase { + AuthenticationConfigurationTestCase { description: "subject cannot be represented by the header transport", + option: "--trusted-proxy-config", configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{"external-café":"00000000-0000-0000-0000-000000000001"}}"#, arguments: &[], expected_error: "subjects must be nonempty, unpadded ASCII header values", }, - ProxyConfigurationTestCase { + AuthenticationConfigurationTestCase { description: "ambiguous password/proxy modes", + option: "--trusted-proxy-config", configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, arguments: &["--password-origin", "https://control.example.com"], expected_error: "trusted-proxy authentication cannot be combined", }, + AuthenticationConfigurationTestCase { + description: "OIDC malformed configuration", + option: "--oidc-config", + configuration: "{", + arguments: &[], + expected_error: "OIDC configuration requires valid JSON", + }, + AuthenticationConfigurationTestCase { + description: "OIDC unknown configuration fields", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"unknown":"synthetic-unrecognized-value"}"#, + arguments: &[], + expected_error: "OIDC configuration requires valid JSON", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects plaintext issuer", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"http://identity.example.com","client_id":"orders-control","identities":{}}"#, + arguments: &[], + expected_error: "OIDC endpoints require HTTPS URLs without credentials", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects plaintext browser origin", + option: "--oidc-config", + configuration: r#"{"origin":"http://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{}}"#, + arguments: &[], + expected_error: "OIDC authentication requires an HTTPS origin", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects missing client secret environment variable", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"client_secret_env":"SQLBUILD_SYNTHETIC_OIDC_MISSING"}"#, + arguments: &[], + expected_error: "OIDC client-secret environment variable is unavailable", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects missing certificate authority", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"certificate_authority":"absent-orders-ca.pem"}"#, + arguments: &[], + expected_error: "could not read OIDC certificate authority", + }, + AuthenticationConfigurationTestCase { + description: "OIDC conflicts with password authentication", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{}}"#, + arguments: &["--password-origin", "https://control.example.com"], + expected_error: "OIDC authentication cannot be combined", + }, ]; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -95,16 +150,19 @@ fn given_invalid_or_conflicting_proxy_configuration_when_a_standalone_server_sta .expect("runtime"); for test_case in test_cases { runtime.block_on(async { - let directory = tempfile::tempdir().expect("invalid proxy installation"); - std::fs::write(directory.path().join("proxy.json"), test_case.configuration) - .expect("configuration input"); + let directory = tempfile::tempdir().expect("invalid identity installation"); + std::fs::write( + directory.path().join("identity.json"), + test_case.configuration, + ) + .expect("configuration input"); let mut arguments = vec![ "server", "--port", "0", "--no-scheduler", - "--trusted-proxy-config", - "proxy.json", + test_case.option, + "identity.json", ]; arguments.extend_from_slice(test_case.arguments); let mut control = start_control_in(directory, &arguments); @@ -120,6 +178,10 @@ fn given_invalid_or_conflicting_proxy_configuration_when_a_standalone_server_sta !log.contains(test_case.configuration), "configuration contents must not be echoed" ); + assert!( + !log.contains("Control API ready:"), + "failed startup must not announce readiness" + ); }); } } diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 73b0341411..517e7a2d83 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -47,8 +47,9 @@ pub(crate) struct ProxyCliTestCase { pub(crate) expected_status: reqwest::StatusCode, } -pub(crate) struct ProxyConfigurationTestCase { +pub(crate) struct AuthenticationConfigurationTestCase { pub(crate) description: &'static str, + pub(crate) option: &'static str, pub(crate) configuration: &'static str, pub(crate) arguments: &'static [&'static str], pub(crate) expected_error: &'static str, diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 2d7004113c..6e944178d1 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -16,6 +16,7 @@ croner.workspace = true flate2.workspace = true futures-util.workspace = true getrandom.workspace = true +openidconnect.workspace = true reqwest.workspace = true rusqlite.workspace = true semver.workspace = true @@ -32,7 +33,9 @@ utoipa.workspace = true uuid.workspace = true [dev-dependencies] +axum-server.workspace = true proptest.workspace = true +rcgen.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs index 043f8046c8..7619f94542 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs @@ -4,12 +4,12 @@ use std::net::SocketAddr; use axum::Router; -use crate::api::_helpers::authentication::{password_sessions, trusted_proxy}; +use crate::api::_helpers::authentication::{oidc, password_sessions, trusted_proxy}; use crate::api::models::{ApiAuthentication, DevelopmentScope}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; -pub(crate) fn configure( +pub(crate) async fn configure( store: &ControlStore, authentication: ApiAuthentication, address: SocketAddr, @@ -18,6 +18,7 @@ pub(crate) fn configure( development_scope, password_origin, trusted_proxy, + oidc, } = authentication; if development_scope.is_some() && password_origin.is_some() { return Err(ControlError::Invalid( @@ -34,9 +35,17 @@ pub(crate) fn configure( "trusted-proxy authentication cannot be combined with disabled or password authentication".into(), )); } + if oidc.is_some() + && (development_scope.is_some() || password_origin.is_some() || trusted_proxy.is_some()) + { + return Err(ControlError::Invalid( + "OIDC authentication cannot be combined with other human authentication modes".into(), + )); + } Ok(( development_scope, password_sessions::router(store, password_origin)? - .merge(trusted_proxy::router(store, trusted_proxy)?), + .merge(trusted_proxy::router(store, trusted_proxy)?) + .merge(oidc::router(store, oidc).await?), )) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs index eb3cf959d7..a8a19eeee6 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod bearer; pub(crate) mod configure; +pub(crate) mod oidc; +pub(crate) mod oidc_provider; pub(crate) mod password_sessions; pub(crate) mod password_transport; pub(crate) mod trusted_proxy; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs new file mode 100644 index 0000000000..80055a7050 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs @@ -0,0 +1,155 @@ +//! Browser-bound authorization-code/PKCE transport authenticates explicitly bound people only. + +use std::sync::Arc; + +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Redirect, Response}; +use axum::routing::get; +use axum::{Extension, Json, Router, middleware}; +use openidconnect::core::CoreAuthenticationFlow; +use openidconnect::{CsrfToken, Nonce, PkceCodeChallenge}; + +use crate::api::_helpers::authentication::oidc_provider; +use crate::api::_helpers::authentication::password_transport::{ + browser_cookie, deny, guard, set_cookie, +}; +use crate::api::constants::OIDC_FLOW_COOKIE; +use crate::api::errors::ApiError; +use crate::api::models::{ + AuthenticationDenial, BrowserOrigin, ExternalIdentityResponse, OidcAuthentication, + OidcCallback, OidcRuntime, +}; +use crate::credentials::constants::OIDC_FLOW_SECONDS; +use crate::credentials::main::authenticate_bound_person::authenticate_bound_person; +use crate::credentials::main::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::{ + claim_oidc_flow::claim_oidc_flow, create_oidc_flow::create_oidc_flow, +}; +use crate::credentials::models::{ + AuditContext, AuthenticationAttempt, ExternalIdentityMechanism, OidcFlowStart, +}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn router( + store: &ControlStore, + settings: Option, +) -> Result, ControlError> { + let Some(settings) = settings else { + return Ok(Router::new()); + }; + let runtime = oidc_provider::configure(settings)?; + oidc_provider::client(&runtime).await?; + Ok(Router::new() + .route("/api/auth/oidc/login", get(login)) + .route("/api/auth/oidc/callback", get(callback)) + .layer(middleware::from_fn_with_state(store.clone(), guard)) + .layer(Extension(BrowserOrigin(runtime.settings.origin.clone()))) + .layer(Extension(Arc::new(runtime)))) +} + +#[utoipa::path(get, path = "/api/auth/oidc/login", operation_id = "loginOidc", + description = "Begins a browser-bound code flow with S256 PKCE; no account provisioning or grants.", + responses((status = 303), (status = 401, body = AuthenticationDenial), (status = 429, body = AuthenticationDenial)))] +pub(crate) async fn login( + State(store): State, + Extension(runtime): Extension>, + Extension(context): Extension, +) -> Result { + let (_, allowed) = + reserve_authentication_attempt(&store, &context, AuthenticationAttempt::IdentityProvider) + .await?; + if !allowed { + let mut response = deny(&store, &context, "oidc_login_throttled").await?; + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + return Ok(response); + } + let client = match oidc_provider::client(&runtime).await { + Ok(client) => client, + Err(_) => return deny(&store, &context, "oidc_provider_unavailable").await, + }; + let (challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); + let (url, state, nonce) = client + .authorize_url( + CoreAuthenticationFlow::AuthorizationCode, + CsrfToken::new_random, + Nonce::new_random, + ) + .set_pkce_challenge(challenge) + .url(); + let flow = OidcFlowStart { + state, + nonce, + pkce_verifier, + browser: CsrfToken::new_random(), + }; + create_oidc_flow(&store, &context, &runtime.configuration_id, &flow).await?; + let mut response = Redirect::to(url.as_str()).into_response(); + set_cookie( + &mut response, + (OIDC_FLOW_COOKIE, flow.browser.secret()), + OIDC_FLOW_SECONDS, + (true, "Lax"), + )?; + Ok(response) +} + +#[utoipa::path(get, path = "/api/auth/oidc/callback", operation_id = "oidcIdentity", + description = "Validates one browser-bound code and returns the explicitly bound person only; no session, bearer, provisioning or grants.", + responses((status = 200, body = ExternalIdentityResponse), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn callback( + State(store): State, + Extension(runtime): Extension>, + Extension(context): Extension, + (headers, Query(request)): (HeaderMap, Query), +) -> Result { + let Some(state) = request.state else { + return deny(&store, &context, "missing_oidc_state").await; + }; + let Some(flow) = claim_oidc_flow( + &store, + &context, + &runtime.configuration_id, + ( + state.secret(), + browser_cookie(&headers, OIDC_FLOW_COOKIE).unwrap_or_default(), + ), + ) + .await? + else { + return deny(&store, &context, "invalid_oidc_browser_or_state").await; + }; + let subject = match request.code { + Some(code) => oidc_provider::verify_code(&runtime, code, flow).await, + None => Err(ControlError::Invalid( + "OIDC authorization was not completed".into(), + )), + }; + let mut response = match subject { + Ok(subject) => match runtime.settings.identities.get(&subject).copied() { + Some(principal) => { + if authenticate_bound_person( + &store, + &context, + principal, + ExternalIdentityMechanism::Oidc, + ) + .await? + .is_some() + { + Json(ExternalIdentityResponse { + principal_id: principal, + }) + .into_response() + } else { + deny(&store, &context, "inactive_oidc_person").await? + } + } + None => deny(&store, &context, "unbound_oidc_subject").await?, + }, + Err(_) => deny(&store, &context, "invalid_oidc_identity").await?, + }; + set_cookie(&mut response, (OIDC_FLOW_COOKIE, ""), 0, (true, "Lax"))?; + Ok(response) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs new file mode 100644 index 0000000000..2976cf2ba5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs @@ -0,0 +1,183 @@ +//! OIDC uses verified HTTPS, bounded bodies and the library's signed issuer/audience/nonce checks. + +use std::time::Duration; + +use openidconnect::core::{CoreClient, CoreProviderMetadata}; +use openidconnect::{ + AccessTokenHash, AuthorizationCode, ClientId, ClientSecret, IssuerUrl, OAuth2TokenResponse, + RedirectUrl, TokenResponse, +}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::api::_helpers::authentication::password_transport::parse_origin; +use crate::api::constants::{HTTPS_SCHEME, OIDC_RESPONSE_BYTES}; +use crate::api::models::{OidcAuthentication, OidcRuntime}; +use crate::api::types::OidcClient; +use crate::credentials::models::StoredOidcFlow; +use crate::store::errors::ControlError; + +pub(crate) fn configure(mut settings: OidcAuthentication) -> Result { + settings.origin = parse_origin(&settings.origin, "OIDC")?.0; + let issuer = https_url(&settings.issuer)?; + if issuer.query().is_some() || issuer.fragment().is_some() { + return Err(ControlError::Invalid( + "OIDC issuer cannot contain a query or fragment".into(), + )); + } + if settings.client_id.trim().is_empty() || settings.identities.keys().any(String::is_empty) { + return Err(ControlError::Invalid( + "OIDC requires a nonempty client ID and nonempty bound subjects".into(), + )); + } + let client_secret = settings + .client_secret_env + .as_ref() + .map(|name| { + let value = std::env::var(name).map_err(|_| { + ControlError::Invalid( + "OIDC client-secret environment variable is unavailable".into(), + ) + })?; + if value.is_empty() { + return Err(ControlError::Invalid( + "OIDC client secret must not be empty".into(), + )); + } + Ok(ClientSecret::new(value)) + }) + .transpose()?; + let mut http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(10)); + if let Some(path) = &settings.certificate_authority { + let bytes = std::fs::read(path).map_err(|_| { + ControlError::Invalid("could not read OIDC certificate authority".into()) + })?; + let certificates = reqwest::Certificate::from_pem_bundle(&bytes) + .map_err(|_| ControlError::Invalid("invalid OIDC PEM certificate authority".into()))?; + if certificates.is_empty() { + return Err(ControlError::Invalid( + "OIDC certificate authority contains no certificates".into(), + )); + } + for certificate in certificates { + http = http.add_root_certificate(certificate); + } + } + let configuration_id = + sha256_hex(&serde_json::to_vec(&settings).map_err(|_| invalid_provider())?); + Ok(OidcRuntime { + settings, + client_secret, + http: http.build().map_err(|_| invalid_provider())?, + configuration_id, + }) +} + +pub(crate) async fn client(runtime: &OidcRuntime) -> Result { + let http = |request| request_provider(runtime.http.clone(), request); + let metadata = CoreProviderMetadata::discover_async( + IssuerUrl::new(runtime.settings.issuer.clone()).map_err(|_| invalid_provider())?, + &http, + ) + .await + .map_err(|_| invalid_provider())?; + https_url(metadata.authorization_endpoint().as_str())?; + https_url( + metadata + .token_endpoint() + .ok_or_else(invalid_provider)? + .as_str(), + )?; + Ok(CoreClient::from_provider_metadata( + metadata, + ClientId::new(runtime.settings.client_id.clone()), + runtime.client_secret.clone(), + ) + .set_redirect_uri( + RedirectUrl::new(format!( + "{}/api/auth/oidc/callback", + runtime.settings.origin + )) + .map_err(|_| invalid_provider())?, + )) +} + +pub(crate) async fn verify_code( + runtime: &OidcRuntime, + code: AuthorizationCode, + flow: StoredOidcFlow, +) -> Result { + let client = client(runtime).await?; + let http = |request| request_provider(runtime.http.clone(), request); + let tokens = client + .exchange_code(code) + .map_err(|_| invalid_provider())? + .set_pkce_verifier(flow.pkce_verifier) + .request_async(&http) + .await + .map_err(|_| invalid_provider())?; + let token = tokens.id_token().ok_or_else(invalid_provider)?; + let verifier = client.id_token_verifier(); + let claims = token + .claims(&verifier, &flow.nonce) + .map_err(|_| invalid_provider())?; + if let Some(expected) = claims.access_token_hash() { + let actual = AccessTokenHash::from_token( + tokens.access_token(), + token.signing_alg().map_err(|_| invalid_provider())?, + token + .signing_key(&verifier) + .map_err(|_| invalid_provider())?, + ) + .map_err(|_| invalid_provider())?; + if actual != *expected { + return Err(invalid_provider()); + } + } + Ok(claims.subject().as_str().to_owned()) +} + +async fn request_provider( + client: reqwest::Client, + request: openidconnect::HttpRequest, +) -> Result { + if request.uri().scheme_str() != Some("https") { + return Err(invalid_provider()); + } + let mut response = client + .execute(reqwest::Request::try_from(request).map_err(|_| invalid_provider())?) + .await + .map_err(|_| invalid_provider())?; + let status = response.status(); + let headers = response.headers().clone(); + let mut bytes: Vec = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|_| invalid_provider())? { + if chunk.len() > OIDC_RESPONSE_BYTES.saturating_sub(bytes.len()) { + return Err(invalid_provider()); + } + bytes.extend_from_slice(&chunk); + } + let mut result = openidconnect::HttpResponse::new(bytes); + *result.status_mut() = status; + *result.headers_mut() = headers; + Ok(result) +} + +fn https_url(value: &str) -> Result { + let url = reqwest::Url::parse(value).map_err(|_| invalid_provider())?; + if url.scheme() != HTTPS_SCHEME + || url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + { + return Err(ControlError::Invalid( + "OIDC endpoints require HTTPS URLs without credentials".into(), + )); + } + Ok(url) +} + +fn invalid_provider() -> ControlError { + ControlError::Invalid("OIDC identity-provider validation failed".into()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 7856d363c2..15b60a9a2e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -7,7 +7,7 @@ use axum::routing::{get, post}; use axum::{Extension, Json, Router, middleware}; use crate::api::_helpers::authentication::password_transport::{ - authenticate, deny, guard, parse_origin, + authenticate, deny, guard, parse_origin, set_cookie, }; use crate::api::constants::{PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE}; use crate::api::errors::ApiError; @@ -16,11 +16,11 @@ use crate::api::models::{ }; use crate::credentials::constants::{LOGIN_WINDOW_SECONDS, PASSWORD_SESSION_SECONDS}; use crate::credentials::main::{ - login_password::login_password, reserve_password_attempt::reserve_password_attempt, + login_password::login_password, reserve_authentication_attempt::reserve_authentication_attempt, revoke_session::revoke_session, set_password::set_password, verify_password::verify_password, }; use crate::credentials::models::{ - AuditContext, PasswordAttempt, PasswordAuthority, PasswordLogin, SessionIdentity, + AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, SessionIdentity, }; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -65,13 +65,13 @@ pub(crate) async fn login( &mut response, (PASSWORD_COOKIE, &session.bearer), PASSWORD_SESSION_SECONDS, - true, + (true, "Strict"), )?; set_cookie( &mut response, (PASSWORD_CSRF_COOKIE, &session.csrf), PASSWORD_SESSION_SECONDS, - false, + (false, "Strict"), )?; Ok(response) } @@ -137,10 +137,10 @@ pub(crate) async fn change_password( Extension(identity): Extension, Json(request): Json, ) -> Result { - let (_, allowed) = reserve_password_attempt( + let (_, allowed) = reserve_authentication_attempt( &store, &context, - PasswordAttempt::Reauthenticate(identity.principal_id), + AuthenticationAttempt::Reauthenticate(identity.principal_id), ) .await?; if !allowed { @@ -166,23 +166,12 @@ pub(crate) async fn change_password( fn clear_cookies() -> Result { let mut response = StatusCode::NO_CONTENT.into_response(); - set_cookie(&mut response, (PASSWORD_COOKIE, ""), 0, true)?; - set_cookie(&mut response, (PASSWORD_CSRF_COOKIE, ""), 0, false)?; + set_cookie(&mut response, (PASSWORD_COOKIE, ""), 0, (true, "Strict"))?; + set_cookie( + &mut response, + (PASSWORD_CSRF_COOKIE, ""), + 0, + (false, "Strict"), + )?; Ok(response) } - -fn set_cookie( - response: &mut Response, - (name, value): (&str, &str), - max_age: i64, - http_only: bool, -) -> Result<(), ApiError> { - let attributes = if http_only { "; HttpOnly" } else { "" }; - let mut cookie = HeaderValue::from_str(&format!( - "{name}={value}; Path=/; Secure; SameSite=Strict; Max-Age={max_age}{attributes}" - )) - .map_err(|_| ControlError::Invalid("could not encode a session cookie".into()))?; - cookie.set_sensitive(true); - response.headers_mut().append(header::SET_COOKIE, cookie); - Ok(()) -} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs index 8d9d8d693c..61d02c1491 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -107,7 +107,7 @@ pub(crate) async fn authenticate( }; let identity = authenticate_session( &store, - session_cookie(request.headers()).unwrap_or_default(), + browser_cookie(request.headers(), PASSWORD_COOKIE).unwrap_or_default(), access, ) .await?; @@ -130,17 +130,17 @@ pub(crate) fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&' values.next().is_none().then_some(value) } -fn session_cookie(headers: &HeaderMap) -> Option<&str> { +pub(crate) fn browser_cookie<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { let mut session = None; for header in headers.get_all(header::COOKIE) { let Ok(value) = header.to_str() else { return None; }; for cookie in value.split(';') { - let Some((name, value)) = cookie.trim().split_once('=') else { + let Some((cookie_name, value)) = cookie.trim().split_once('=') else { continue; }; - if name == PASSWORD_COOKIE { + if cookie_name == name { if session.is_some() { return None; } @@ -151,6 +151,22 @@ fn session_cookie(headers: &HeaderMap) -> Option<&str> { session } +pub(crate) fn set_cookie( + response: &mut Response, + (name, value): (&str, &str), + max_age: i64, + (http_only, same_site): (bool, &str), +) -> Result<(), ApiError> { + let attributes = if http_only { "; HttpOnly" } else { "" }; + let mut cookie = HeaderValue::from_str(&format!( + "{name}={value}; Path=/; Secure; SameSite={same_site}; Max-Age={max_age}{attributes}" + )) + .map_err(|_| ControlError::Invalid("could not encode a session cookie".into()))?; + cookie.set_sensitive(true); + response.headers_mut().append(header::SET_COOKIE, cookie); + Ok(()) +} + pub(crate) async fn deny( store: &ControlStore, context: &AuditContext, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs index 7f5f2191e3..916483eed8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs @@ -2,24 +2,23 @@ use std::sync::Arc; -use axum::extract::State; -use axum::http::{HeaderMap, HeaderValue}; -use axum::response::{IntoResponse, Response}; -use axum::routing::get; -use axum::{Extension, Json, Router, middleware}; -use serde_json::json; - use crate::api::_helpers::authentication::password_transport::{ deny, guard, parse_origin, single_header, }; use crate::api::constants::PROXY_USER_HEADER; use crate::api::errors::ApiError; -use crate::api::models::{AuthenticationDenial, ProxyIdentityResponse, TrustedProxyAuthentication}; -use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::AuditContext; +use crate::api::models::{ + AuthenticationDenial, ExternalIdentityResponse, TrustedProxyAuthentication, +}; +use crate::credentials::main::authenticate_bound_person::authenticate_bound_person; +use crate::credentials::models::{AuditContext, ExternalIdentityMechanism}; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::ControlStore; +use axum::extract::State; +use axum::http::{HeaderMap, HeaderValue}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{Extension, Json, Router, middleware}; pub(crate) fn router( store: &ControlStore, @@ -61,11 +60,11 @@ pub(crate) fn router( #[utoipa::path(get, path = "/api/auth/proxy", operation_id = "proxyIdentity", description = "Requires a configured socket peer and an upstream-replaced X-SQB-User subject. Does not issue sessions or grant permissions.", security(("proxy" = [])), - responses((status = 200, body = ProxyIdentityResponse), (status = 401, body = AuthenticationDenial)))] + responses((status = 200, body = ExternalIdentityResponse), (status = 401, body = AuthenticationDenial)))] pub(crate) async fn identity( State(store): State, Extension(settings): Extension>, - Extension(mut context): Extension, + Extension(context): Extension, headers: HeaderMap, ) -> Result { if !context @@ -80,30 +79,16 @@ pub(crate) async fn identity( let Some(principal_id) = principal else { return deny(&store, &context, "unbound_proxy_subject").await; }; - let transaction = begin(&store, TransactionMode::Write).await?; - let (transaction, rows) = query(transaction, - "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND kind='person' AND disabled_at IS NULL", - vec![Parameter::Id(principal_id)]).await?; - if rows.is_empty() { - commit(transaction).await?; + if authenticate_bound_person( + &store, + &context, + principal_id, + ExternalIdentityMechanism::TrustedProxy, + ) + .await? + .is_none() + { return deny(&store, &context, "inactive_proxy_person").await; } - context.actor = format!("principal:{principal_id}"); - context.granting_role = "trusted-proxy".into(); - let identity = ProxyIdentityResponse { principal_id }; - commit( - append_audit( - transaction, - &context, - (context.request_id, "proxy.authenticated"), - ( - json!({"principal_id": principal_id}), - json!(null), - json!(&identity), - ), - ) - .await?, - ) - .await?; - Ok(Json(identity).into_response()) + Ok(Json(ExternalIdentityResponse { principal_id }).into_response()) } diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index 1ee030772c..b4f810e892 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -7,5 +7,7 @@ pub(crate) const PASSWORD_COOKIE: &str = "__Host-sqb-session"; pub(crate) const PASSWORD_CSRF_COOKIE: &str = "__Host-sqb-csrf"; pub(crate) const PASSWORD_CSRF_HEADER: &str = "x-sqb-csrf"; pub(crate) const PROXY_USER_HEADER: &str = "x-sqb-user"; +pub(crate) const OIDC_FLOW_COOKIE: &str = "__Host-sqb-oidc"; +pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024; pub(crate) const HTTPS_SCHEME: &str = "https"; pub(crate) const ROOT_PATH: &str = "/"; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index a1b98c1af5..7bc810083d 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -16,10 +16,14 @@ pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, authentication: ApiAuthentication, - shutdown: impl Future + Send + 'static, + (ready, shutdown): ( + impl FnOnce() + Send + 'static, + impl Future + Send + 'static, + ), ) -> Result<(), ControlError> { let address = listener.local_addr()?; - let (development_scope, password_routes) = configure(&store, authentication, address)?; + let (development_scope, authentication_routes) = + configure(&store, authentication, address).await?; migrate_store(&store).await?; let (sender, listener_changes) = subscribe_changes(&store).await?; let (stop, stopped) = tokio::sync::watch::channel(false); @@ -28,7 +32,7 @@ pub async fn serve_api( stopped, }; let application = router(&store, feed) - .merge(password_routes) + .merge(authentication_routes) .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), @@ -43,6 +47,7 @@ pub async fn serve_api( shutdown.await; stop.send_replace(true); }); + ready(); tokio::select! { result = async { server.await } => { result?; Ok(()) }, result = changes::relay(store, listener_changes, sender) => result, diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index 524d320a8c..988baae351 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub mod models; +pub(crate) mod types; #[cfg(test)] #[path = "../../tests/api/test_contracts.rs"] diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index bebb162ccb..698a7aa568 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; +use crate::api::_helpers::authentication::oidc; use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::trusted_proxy; use crate::api::_helpers::resources::{ @@ -16,6 +17,8 @@ use crate::api::_helpers::resources::{ password_sessions::logout, password_sessions::change_password, trusted_proxy::identity, + oidc::login, + oidc::callback, changes::subscribe, location_health::report, location_health::recent, @@ -103,6 +106,7 @@ pub struct ApiAuthentication { pub development_scope: Option, pub password_origin: Option, pub trusted_proxy: Option, + pub oidc: Option, } #[derive(Clone, Debug)] @@ -117,8 +121,34 @@ pub struct TrustedProxyAuthentication { pub identities: std::collections::BTreeMap, } +/// Exact issuer/subject bindings authenticate existing people without provisioning or grants. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OidcAuthentication { + pub origin: String, + pub issuer: String, + pub client_id: String, + pub client_secret_env: Option, + pub certificate_authority: Option, + pub identities: std::collections::BTreeMap, +} + +#[derive(Debug)] +pub(crate) struct OidcRuntime { + pub(crate) settings: OidcAuthentication, + pub(crate) client_secret: Option, + pub(crate) http: reqwest::Client, + pub(crate) configuration_id: String, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OidcCallback { + pub(crate) state: Option, + pub(crate) code: Option, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] -pub struct ProxyIdentityResponse { +pub struct ExternalIdentityResponse { pub principal_id: uuid::Uuid, } diff --git a/crates/sqlbuild-orchestration/src/api/types.rs b/crates/sqlbuild-orchestration/src/api/types.rs new file mode 100644 index 0000000000..2a77575870 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/types.rs @@ -0,0 +1,12 @@ +//! The provider's discovered authorization and token endpoints retain their typed availability. + +use openidconnect::{EndpointMaybeSet, EndpointNotSet, EndpointSet}; + +pub(crate) type OidcClient = openidconnect::core::CoreClient< + EndpointSet, + EndpointNotSet, + EndpointNotSet, + EndpointNotSet, + EndpointMaybeSet, + EndpointMaybeSet, +>; diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs index d21a3bfd42..26dc2378bc 100644 --- a/crates/sqlbuild-orchestration/src/credentials/constants.rs +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -8,6 +8,7 @@ pub(crate) const MAX_PASSWORD_BYTES: usize = 1024; pub(crate) const PASSWORD_ALGORITHM: &str = "argon2id"; pub(crate) const PASSWORD_KDF_SLOTS: usize = 4; pub(crate) const PASSWORD_SESSION_SECONDS: i64 = 3600; +pub(crate) const OIDC_FLOW_SECONDS: i64 = 300; pub(crate) const LOGIN_WINDOW_SECONDS: i64 = 60; pub(crate) const LOGIN_USER_LIMIT: u64 = 5; pub(crate) const LOGIN_PEER_LIMIT: u64 = 20; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs new file mode 100644 index 0000000000..c242dff68c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs @@ -0,0 +1,47 @@ +//! Federation proves identity only; explicit bindings never provision accounts or grant roles. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, ExternalIdentityMechanism}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn authenticate_bound_person( + store: &ControlStore, + context: &AuditContext, + principal_id: Uuid, + mechanism: ExternalIdentityMechanism, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, rows) = query(transaction, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND kind='person' AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)]).await?; + if rows.is_empty() { + commit(transaction).await?; + return Ok(None); + } + let (action, granting_role) = match mechanism { + ExternalIdentityMechanism::TrustedProxy => ("proxy.authenticated", "trusted-proxy"), + ExternalIdentityMechanism::Oidc => ("oidc.authenticated", "oidc"), + }; + let context = AuditContext { + actor: format!("principal:{principal_id}"), + granting_role: granting_role.into(), + ..context.clone() + }; + let identity = json!({"principal_id": principal_id}); + commit( + append_audit( + transaction, + &context, + (context.request_id, action), + (identity.clone(), json!(null), identity), + ) + .await?, + ) + .await?; + Ok(Some(principal_id)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs b/crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs new file mode 100644 index 0000000000..b02f92452c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs @@ -0,0 +1,66 @@ +//! Atomically consume browser-bound flow state before any identity-provider exchange. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::secret_matches; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, StoredOidcFlow}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn claim_oidc_flow( + store: &ControlStore, + context: &AuditContext, + configuration_id: &str, + (state, browser): (&str, &str), +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT flow_id, browser_hash, nonce, pkce_verifier FROM sqb_oidc_login_flows \ + WHERE state_hash=$1 AND configuration_id=$2 AND expires_at>$3", + vec![ + Parameter::Text(sha256_hex(state.as_bytes())), + Parameter::Text(configuration_id.into()), + Parameter::Time(now), + ], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let flow: StoredOidcFlow = decode_row(row)?; + if !secret_matches(browser, &flow.browser_hash) { + commit(transaction).await?; + return Ok(None); + } + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_oidc_login_flows WHERE flow_id=$1", + vec![Parameter::Id(flow.flow_id)], + ) + .await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&flow.flow_id, b"consumed"), + "oidc.flow.consumed", + ), + ( + json!({"flow_id": flow.flow_id}), + json!({"pending": true}), + json!({"pending": false}), + ), + ) + .await?, + ) + .await?; + Ok(Some(flow)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs new file mode 100644 index 0000000000..83eb7afcf8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs @@ -0,0 +1,47 @@ +//! Create one-use browser-bound PKCE attempts with durable audit evidence. + +use crate::credentials::constants::OIDC_FLOW_SECONDS; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, OidcFlowStart}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use chrono::Duration; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +pub(crate) async fn create_oidc_flow( + store: &ControlStore, + context: &AuditContext, + configuration_id: &str, + flow: &OidcFlowStart, +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "DELETE FROM sqb_oidc_login_flows WHERE flow_id IN \ + (SELECT flow_id FROM sqb_oidc_login_flows WHERE expires_at<=$1 ORDER BY expires_at LIMIT 1000)", + vec![Parameter::Time(now)]).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_oidc_login_flows (flow_id, configuration_id, state_hash, browser_hash, nonce, pkce_verifier, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![Parameter::Id(context.request_id), Parameter::Text(configuration_id.into()), + Parameter::Text(sha256_hex(flow.state.secret().as_bytes())), + Parameter::Text(sha256_hex(flow.browser.secret().as_bytes())), + Parameter::Text(flow.nonce.secret().clone()), Parameter::Text(flow.pkce_verifier.secret().clone()), + Parameter::Time(now + Duration::seconds(OIDC_FLOW_SECONDS))]).await?; + commit( + append_audit( + transaction, + context, + (context.request_id, "oidc.flow.created"), + ( + json!({"flow_id": context.request_id}), + json!(null), + json!({"pending": true}), + ), + ) + .await?, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs index e7b9254e82..2f86ab6cab 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs @@ -5,9 +5,9 @@ use uuid::Uuid; use crate::credentials::_helpers::password_sessions::issue_session; use crate::credentials::constants::{LOGIN_USERNAME_BYTES, MAX_PASSWORD_BYTES}; use crate::credentials::main::record_authentication_denial::record_authentication_denial; -use crate::credentials::main::reserve_password_attempt::reserve_password_attempt; +use crate::credentials::main::reserve_authentication_attempt::reserve_authentication_attempt; use crate::credentials::main::verify_password::verify_password; -use crate::credentials::models::{AuditContext, PasswordAttempt, PasswordLogin}; +use crate::credentials::models::{AuditContext, AuthenticationAttempt, PasswordLogin}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -20,7 +20,8 @@ pub async fn login_password( let bounded = username.len() <= LOGIN_USERNAME_BYTES && password.len() <= MAX_PASSWORD_BYTES; let username = if bounded { username } else { "" }; let (attempt_id, allowed) = - reserve_password_attempt(store, context, PasswordAttempt::Login(username)).await?; + reserve_authentication_attempt(store, context, AuthenticationAttempt::Login(username)) + .await?; if !allowed { return Ok(PasswordLogin::Denied { decision_id: attempt_id, diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index eb9b124133..ed61a1d2ae 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,13 +1,16 @@ pub(crate) mod append_audit; +pub(crate) mod authenticate_bound_person; pub mod authenticate_session; pub mod authenticate_token; +pub(crate) mod claim_oidc_flow; pub mod create_location; +pub(crate) mod create_oidc_flow; pub mod create_principal; pub mod development_location; pub mod issue_token; pub mod login_password; pub mod record_authentication_denial; -pub mod reserve_password_attempt; +pub mod reserve_authentication_attempt; pub mod revoke_session; pub mod revoke_token; pub mod rotate_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs b/crates/sqlbuild-orchestration/src/credentials/main/reserve_authentication_attempt.rs similarity index 78% rename from crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs rename to crates/sqlbuild-orchestration/src/credentials/main/reserve_authentication_attempt.rs index 23fa3711b2..434b0a4eb3 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/reserve_password_attempt.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/reserve_authentication_attempt.rs @@ -1,4 +1,4 @@ -//! Serialized reservations bound concurrent password work across API replicas. +//! Serialized reservations bound authentication work across API replicas. use chrono::Duration; use serde_json::json; @@ -6,25 +6,30 @@ use uuid::Uuid; use crate::credentials::constants::{LOGIN_PEER_LIMIT, LOGIN_USER_LIMIT, LOGIN_WINDOW_SECONDS}; use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{AuditContext, PasswordAttempt}; +use crate::credentials::models::{AuditContext, AuthenticationAttempt}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -pub async fn reserve_password_attempt( +pub async fn reserve_authentication_attempt( store: &ControlStore, context: &AuditContext, - attempt: PasswordAttempt<'_>, + attempt: AuthenticationAttempt<'_>, ) -> Result<(Uuid, bool), ControlError> { - let identity_key = match attempt { - PasswordAttempt::Login(username) => format!("login:{username}"), - PasswordAttempt::Reauthenticate(id) => format!("reauthenticate:{id}"), - }; let peer = context .ip_address .ok_or_else(|| ControlError::Invalid("login requires a trusted peer address".into()))? .to_canonical() .to_string(); + let (identity_key, identity_limit) = match attempt { + AuthenticationAttempt::Login(username) => (format!("login:{username}"), LOGIN_USER_LIMIT), + AuthenticationAttempt::Reauthenticate(id) => { + (format!("reauthenticate:{id}"), LOGIN_USER_LIMIT) + } + AuthenticationAttempt::IdentityProvider => { + (format!("identity-provider:{peer}"), LOGIN_PEER_LIMIT) + } + }; let transaction = begin(store, TransactionMode::Write).await?; let now = transaction.now; let (transaction, _) = query(transaction, @@ -37,7 +42,7 @@ pub async fn reserve_password_attempt( vec![Parameter::Text(identity_key.clone()), Parameter::Text(peer.clone()), Parameter::Time(now - Duration::seconds(LOGIN_WINDOW_SECONDS))]).await?; let allowed = rows[0]["user_count"] .as_u64() - .is_some_and(|count| count < LOGIN_USER_LIMIT) + .is_some_and(|count| count < identity_limit) && rows[0]["peer_count"] .as_u64() .is_some_and(|count| count < LOGIN_PEER_LIMIT); diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index 6ac4e3d876..7c9157a42e 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -96,9 +96,32 @@ pub enum PasswordLogin { Denied { decision_id: Uuid, throttled: bool }, } -pub enum PasswordAttempt<'a> { +pub enum AuthenticationAttempt<'a> { Login(&'a str), Reauthenticate(Uuid), + IdentityProvider, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum ExternalIdentityMechanism { + TrustedProxy, + Oidc, +} + +#[derive(Debug)] +pub(crate) struct OidcFlowStart { + pub(crate) state: openidconnect::CsrfToken, + pub(crate) browser: openidconnect::CsrfToken, + pub(crate) nonce: openidconnect::Nonce, + pub(crate) pkce_verifier: openidconnect::PkceCodeVerifier, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredOidcFlow { + pub(crate) flow_id: Uuid, + pub(crate) browser_hash: String, + pub(crate) nonce: openidconnect::Nonce, + pub(crate) pkce_verifier: openidconnect::PkceCodeVerifier, } #[derive(Debug, Deserialize)] diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 94306d2b83..cc627ae85b 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -21,6 +21,7 @@ pub(crate) const NODE_TRANSITIONS: &str = include_str!("../node_transitions.sql" pub(crate) const PASSWORD_CREDENTIALS: &str = include_str!("../password_credentials.sql"); pub(crate) const PASSWORD_SESSIONS: &str = include_str!("../password_sessions.sql"); pub(crate) const AUTHENTICATION_REVISION: &str = include_str!("../authentication_revision.sql"); +pub(crate) const OIDC_LOGIN_FLOWS: &str = include_str!("../oidc_login_flows.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index dc1fc3e4d8..e63bfc3c31 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,9 +5,9 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, - LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, - render, verify_migrations, + LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, + PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, + WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -45,6 +45,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("password_credentials", PASSWORD_CREDENTIALS), ("password_sessions", PASSWORD_SESSIONS), ("authentication_revision", AUTHENTICATION_REVISION), + ("oidc_login_flows", OIDC_LOGIN_FLOWS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/oidc_login_flows.sql b/crates/sqlbuild-orchestration/src/store/oidc_login_flows.sql new file mode 100644 index 0000000000..10cdb76422 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/oidc_login_flows.sql @@ -0,0 +1,10 @@ +CREATE TABLE sqb_oidc_login_flows ( + flow_id {uuid} PRIMARY KEY, + configuration_id TEXT NOT NULL, + state_hash TEXT NOT NULL UNIQUE, + browser_hash TEXT NOT NULL, + nonce TEXT NOT NULL, + pkce_verifier TEXT NOT NULL, + expires_at {time} NOT NULL +); +CREATE INDEX sqb_oidc_login_flows_expiry ON sqb_oidc_login_flows(expires_at); diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 8fca2ac42a..4f5b42e081 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -19,9 +19,14 @@ pub(crate) async fn start_api_with_authentication( let url = format!("http://{}", listener.local_addr().expect("API address")); let (shutdown, signal) = tokio::sync::oneshot::channel(); let store = store.clone(); - let task = tokio::spawn(serve_api(store, listener, authentication, async { - let _ = signal.await; - })); + let task = tokio::spawn(serve_api( + store, + listener, + authentication, + (|| {}, async { + let _ = signal.await; + }), + )); let client = reqwest::Client::builder() .timeout(Duration::from_secs(1)) .build() diff --git a/crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs b/crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs new file mode 100644 index 0000000000..4543161662 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs @@ -0,0 +1,259 @@ +//! Real TLS discovery, JWKS, authorization and code exchange for native HTTP acceptance. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::extract::{Query, State}; +use axum::response::Redirect; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use chrono::{Duration, Utc}; +use openidconnect::core::{ + CoreEdDsaPrivateSigningKey, CoreIdToken, CoreIdTokenClaims, CoreJwsSigningAlgorithm, +}; +use openidconnect::{ + AccessToken, Audience, EmptyAdditionalClaims, IssuerUrl, JsonWebKeyId, Nonce, + PkceCodeChallenge, PkceCodeVerifier, PrivateSigningKey, StandardClaims, SubjectIdentifier, +}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::oidc_test_types::{BrowserFlow, ProviderFault, ProviderState, TestProvider}; + +pub(crate) async fn start_provider() -> TestProvider { + let directory = tempfile::tempdir().expect("disposable provider certificates"); + let certificate = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("fresh TLS key and certificate"); + let pem = certificate.cert.pem(); + let authority = directory.path().join("orders-ca.pem"); + std::fs::write(&authority, &pem).expect("fixture authority"); + let tls = axum_server::tls_rustls::RustlsConfig::from_pem( + pem.as_bytes().to_vec(), + certificate.signing_key.serialize_pem().into_bytes(), + ) + .await + .expect("TLS configuration"); + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("provider socket"); + listener + .set_nonblocking(true) + .expect("asynchronous TLS listener"); + let port = listener.local_addr().expect("provider address").port(); + let key = |id: &str| { + let key = rcgen::KeyPair::generate_for(&rcgen::PKCS_ED25519).expect("fresh signing key"); + Arc::new( + CoreEdDsaPrivateSigningKey::from_ed25519_pem( + &key.serialize_pem(), + Some(JsonWebKeyId::new(id.into())), + ) + .expect("OIDC signing key"), + ) + }; + let state = ProviderState { + issuer: format!("https://localhost:{port}"), + keys: Arc::new(BTreeMap::from([ + ("orders".into(), key("orders")), + ("customers".into(), key("customers")), + ])), + published_key: Arc::new(Mutex::new("orders".into())), + codes: Arc::new(Mutex::new(BTreeMap::new())), + fault: Arc::new(Mutex::new(ProviderFault::default())), + subject: Arc::new(Mutex::new("orders-person".into())), + }; + let router = Router::new() + .route("/.well-known/openid-configuration", get(discovery)) + .route("/jwks", get(jwks)) + .route("/authorize", get(authorize)) + .route("/token", post(token)) + .with_state(state.clone()); + let handle = axum_server::Handle::new(); + let server = axum_server::from_tcp_rustls(listener, tls) + .expect("TLS listener") + .handle(handle.clone()); + let task = tokio::spawn(server.serve(router.into_make_service())); + let client = reqwest::Client::builder() + .add_root_certificate(reqwest::Certificate::from_pem(pem.as_bytes()).expect("root")) + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("verified TLS client"); + TestProvider { + state, + authority, + client, + handle, + task, + directory, + } +} + +pub(crate) async fn stop_provider(provider: TestProvider) { + provider + .handle + .graceful_shutdown(Some(std::time::Duration::from_secs(1))); + provider + .task + .await + .expect("join provider") + .expect("provider shutdown"); + provider + .directory + .close() + .expect("remove disposable provider keys"); +} + +async fn discovery(State(state): State) -> Json { + Json(json!({ + "issuer": state.issuer, + "authorization_endpoint": format!("{}/authorize", state.issuer), + "token_endpoint": format!("{}/token", state.issuer), + "jwks_uri": format!("{}/jwks", state.issuer), + "response_types_supported": ["code"], "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["EdDSA"], + "code_challenge_methods_supported": ["S256"] + })) +} + +async fn jwks(State(state): State) -> Json { + let key = state.published_key.lock().expect("published key"); + Json(json!({"keys": [state.keys[key.as_str()].as_verification_key()]})) +} + +async fn authorize( + State(state): State, + Query(request): Query>, +) -> Redirect { + assert_eq!(request["client_id"], "orders-control"); + assert_eq!(request["response_type"], "code"); + assert_eq!(request["code_challenge_method"], "S256"); + assert!( + request["scope"] + .split_whitespace() + .any(|scope| scope == "openid") + ); + let code = Uuid::now_v7().to_string(); + let mut callback = reqwest::Url::parse(&request["redirect_uri"]).expect("callback URI"); + callback + .query_pairs_mut() + .append_pair("code", &code) + .append_pair("state", &request["state"]); + state.codes.lock().expect("code lock").insert(code, request); + Redirect::to(callback.as_str()) +} + +async fn token(State(state): State, body: String) -> Json { + let form = reqwest::Url::parse(&format!("https://provider.example.com/?{body}")) + .expect("token form") + .query_pairs() + .into_owned() + .collect::>(); + assert_eq!(form["grant_type"], "authorization_code"); + let request = state + .codes + .lock() + .expect("code lock") + .remove(&form["code"]) + .expect("one-use authorization code"); + assert_eq!(form["redirect_uri"], request["redirect_uri"]); + let challenge = PkceCodeChallenge::from_code_verifier_sha256(&PkceCodeVerifier::new( + form["code_verifier"].clone(), + )); + assert_eq!(challenge.as_str(), request["code_challenge"]); + let fault = state.fault.lock().expect("fault lock").clone(); + let claims = CoreIdTokenClaims::new( + IssuerUrl::new(state.issuer.clone()).expect("issuer"), + vec![Audience::new("orders-control".into())], + Utc::now() + Duration::seconds(300), + Utc::now(), + StandardClaims::new(SubjectIdentifier::new( + state.subject.lock().expect("subject").clone(), + )), + EmptyAdditionalClaims {}, + ) + .set_nonce(Some(Nonce::new(request["nonce"].clone()))); + let mut claims = serde_json::to_value(claims).expect("claims JSON"); + claims + .as_object_mut() + .expect("claims object") + .extend(fault.claims); + let claims = serde_json::from_value::(claims).expect("fixture claims"); + let signing = &state.keys[fault.signing_key.as_deref().unwrap_or("orders")]; + let id_token = CoreIdToken::new( + claims, + signing.as_ref(), + CoreJwsSigningAlgorithm::EdDsa, + Some(&AccessToken::new("orders-access".into())), + None, + ) + .expect("signed token"); + let mut response = json!({"token_type":"Bearer", "expires_in":300, + "access_token": "orders-access", + "id_token": id_token.to_string()}); + response + .as_object_mut() + .expect("response") + .extend(fault.response); + for field in fault.remove_fields { + response.as_object_mut().expect("response").remove(&field); + } + Json(response) +} + +pub(crate) async fn login(provider: &TestProvider, url: &str) -> BrowserFlow { + let response = provider + .client + .get(format!("{url}/api/auth/oidc/login")) + .header(reqwest::header::HOST, "control.example.com") + .send() + .await + .expect("login"); + assert_eq!(response.status(), reqwest::StatusCode::SEE_OTHER); + let cookie = response.headers()[reqwest::header::SET_COOKIE] + .to_str() + .expect("cookie"); + for attribute in [ + "Secure", + "HttpOnly", + "SameSite=Lax", + "Path=/", + "Max-Age=300", + ] { + assert!(cookie.contains(attribute), "{attribute}"); + } + assert!(!cookie.contains("Domain=")); + let cookie = cookie.split(';').next().expect("browser binding").into(); + let authorize = response.headers()[reqwest::header::LOCATION] + .to_str() + .expect("authorization URL"); + let response = provider + .client + .get(authorize) + .send() + .await + .expect("provider authorize"); + assert_eq!(response.status(), reqwest::StatusCode::SEE_OTHER); + let callback = reqwest::Url::parse( + response.headers()[reqwest::header::LOCATION] + .to_str() + .expect("callback"), + ) + .expect("callback URL"); + BrowserFlow { cookie, callback } +} + +pub(crate) async fn callback( + provider: &TestProvider, + url: &str, + flow: &BrowserFlow, +) -> reqwest::Response { + let mut endpoint = reqwest::Url::parse(url).expect("API URL"); + endpoint.set_path(flow.callback.path()); + endpoint.set_query(flow.callback.query()); + provider + .client + .get(endpoint) + .header(reqwest::header::HOST, "control.example.com") + .header(reqwest::header::COOKIE, &flow.cookie) + .send() + .await + .expect("callback response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs b/crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs new file mode 100644 index 0000000000..38a686f66d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs @@ -0,0 +1,39 @@ +//! Disposable HTTPS identity-provider state; signing keys are generated for each fixture. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use openidconnect::core::CoreEdDsaPrivateSigningKey; + +#[derive(Clone)] +pub(crate) struct ProviderState { + pub(crate) issuer: String, + pub(crate) keys: Arc>>, + pub(crate) published_key: Arc>, + pub(crate) codes: Arc>>>, + pub(crate) fault: Arc>, + pub(crate) subject: Arc>, +} + +pub(crate) struct TestProvider { + pub(crate) state: ProviderState, + pub(crate) authority: std::path::PathBuf, + pub(crate) client: reqwest::Client, + pub(crate) handle: axum_server::Handle, + pub(crate) task: tokio::task::JoinHandle>, + pub(crate) directory: tempfile::TempDir, +} + +pub(crate) struct BrowserFlow { + pub(crate) cookie: String, + pub(crate) callback: reqwest::Url, +} + +#[derive(Clone, Default, serde::Deserialize)] +#[serde(default)] +pub(crate) struct ProviderFault { + pub(crate) claims: serde_json::Map, + pub(crate) response: serde_json::Map, + pub(crate) remove_fields: Vec, + pub(crate) signing_key: Option, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 1cac36cbb2..6b50cce76a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -4,6 +4,10 @@ mod changes_helpers; #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "oidc/helpers.rs"] +mod oidc_helpers; +#[path = "oidc/test_types.rs"] +mod oidc_test_types; #[path = "output/helpers.rs"] mod output_helpers; #[path = "recall/helpers.rs"] @@ -18,6 +22,8 @@ mod test_development; mod test_location_health; #[path = "test_location_work.rs"] mod test_location_work; +#[path = "test_oidc.rs"] +mod test_oidc; #[path = "test_passwords.rs"] mod test_passwords; #[path = "test_proxy.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 9686e6f45e..9796d4536c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -49,7 +49,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable for (version, location_cleanup) in [(3, ""), (2, "DROP TABLE sqb_location_instances;")] { execute(store, &format!( - "DROP TABLE sqb_authentication_revision; \ + "DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; \ DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ diff --git a/crates/sqlbuild-orchestration/tests/api/test_oidc.rs b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs new file mode 100644 index 0000000000..13b16da7bd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs @@ -0,0 +1,352 @@ +//! Signed HTTPS provider exchanges exercise PKCE, browser binding and replica-safe one-use state. + +use std::collections::BTreeMap; + +use reqwest::{StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, OidcAuthentication}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::oidc_helpers::{callback, login, start_provider, stop_provider}; +use crate::api::tests::oidc_test_types::{BrowserFlow, ProviderFault}; +use crate::api::tests::test_types::{OidcApiTestCase, OidcThrottleTestCase}; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_oidc_codes_when_verified_across_replicas_then_only_bound_live_people_authenticate() { + let test_cases = [ + OidcApiTestCase { + description: "SQLite OIDC boundary", + backend: Backend::Sqlite, + expected_authentications: 5, + }, + OidcApiTestCase { + description: "PostgreSQL OIDC boundary", + backend: Backend::Postgres, + expected_authentications: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let provider = start_provider().await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json"); + let inputs: Value = serde_json::from_str( + &std::fs::read_to_string(path).expect("DuckShop OIDC fixture"), + ) + .expect("DuckShop OIDC cases"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = + create_principal(store, &context, ("orders-person", PrincipalKind::Person)) + .await + .expect("person"); + let service = create_principal( + store, + &context, + ("orders-service", PrincipalKind::ServiceAccount), + ) + .await + .expect("service"); + let settings = OidcAuthentication { + origin: inputs["origin"].as_str().expect("origin").into(), + issuer: provider.state.issuer.clone(), + client_id: "orders-control".into(), + client_secret_env: None, + certificate_authority: Some(provider.authority.clone()), + identities: BTreeMap::from([ + ("orders-person".into(), person), + ("orders-service".into(), service), + ("absent-person".into(), Uuid::now_v7()), + ]), + }; + let api = start_api_with_authentication( + store, + ApiAuthentication { + oidc: Some(settings.clone()), + ..Default::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + oidc: Some(settings), + ..Default::default() + }, + ) + .await; + let flow = login(&provider, &api.url).await; + let mut wrong = BrowserFlow { + cookie: "__Host-sqb-oidc=wrong-browser".into(), + callback: flow.callback.clone(), + }; + assert_eq!( + callback(&provider, &other.url, &wrong).await.status(), + StatusCode::UNAUTHORIZED + ); + wrong.cookie = flow.cookie.clone(); + wrong + .callback + .query_pairs_mut() + .clear() + .append_pair("state", "wrong-state") + .append_pair("code", "wrong-code"); + assert_eq!( + callback(&provider, &other.url, &wrong).await.status(), + StatusCode::UNAUTHORIZED + ); + let response = callback(&provider, &other.url, &flow).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert!( + response.headers()[header::SET_COOKIE] + .to_str() + .expect("cleared cookie") + .contains("Max-Age=0") + ); + assert_eq!( + response.json::().await.expect("identity"), + json!({"principal_id":person}) + ); + assert_eq!( + callback(&provider, &api.url, &flow).await.status(), + StatusCode::UNAUTHORIZED + ); + for fault in inputs["invalid_tokens"].as_array().expect("token cases") { + *provider.state.fault.lock().expect("fault") = + serde_json::from_value(fault.clone()).expect("token fault"); + let flow = login(&provider, &api.url).await; + let response = callback(&provider, &other.url, &flow).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{fault}"); + let body = response.text().await.expect("sanitized denial"); + assert!(!body.contains("orders-access")); + assert!(!body.contains("code_verifier")); + } + *provider.state.fault.lock().expect("fault") = ProviderFault::default(); + for subject in inputs["ineligible_subjects"].as_array().expect("subjects") { + *provider.state.subject.lock().expect("subject") = + subject.as_str().expect("subject name").into(); + let flow = login(&provider, &api.url).await; + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::UNAUTHORIZED, + "{subject}" + ); + } + *provider.state.subject.lock().expect("subject") = "orders-person".into(); + let flow = login(&provider, &api.url).await; + *provider.state.published_key.lock().expect("published key") = "customers".into(); + provider.state.fault.lock().expect("fault").signing_key = Some("customers".into()); + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::OK, + "a key rotated after authorization is fetched at callback" + ); + *provider.state.published_key.lock().expect("published key") = "orders".into(); + *provider.state.fault.lock().expect("fault") = ProviderFault::default(); + for (action, condition, expected_retry) in [ + ( + "oidc.flow.consumed", + "WHEN NEW.action='oidc.flow.consumed'", + StatusCode::OK, + ), + ( + "oidc.authenticated", + "WHEN NEW.action='oidc.authenticated'", + StatusCode::UNAUTHORIZED, + ), + ] { + let flow = login(&provider, &api.url).await; + let point = TransactionFault { + description: "OIDC audit publication failure", + table: "sqb_security_audit", + operation: "INSERT", + condition, + sqlite_body: "SELECT RAISE(ABORT, 'synthetic OIDC audit failure')", + postgres_body: "RAISE EXCEPTION 'synthetic OIDC audit failure'", + }; + install_fault(store, &point).await; + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &point).await; + let retry = callback(&provider, &api.url, &flow).await; + assert_eq!(retry.status(), expected_retry, "{action}"); + } + let flow = login(&provider, &api.url).await; + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::OK, + "fresh login recovers after an audited-identity publication failure" + ); + let flow = login(&provider, &api.url).await; + let (left, right) = tokio::join!( + callback(&provider, &api.url, &flow), + callback(&provider, &other.url, &flow) + ); + let mut statuses = [left.status().as_u16(), right.status().as_u16()]; + statuses.sort(); + assert_eq!(statuses, [200, 401], "one concurrent callback succeeds"); + let flow = login(&provider, &api.url).await; + let transaction = begin(store, TransactionMode::Write).await.expect("disable"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(now), Parameter::Id(person)], + ) + .await + .expect("live disable"); + commit(transaction).await.expect("disable commit"); + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::UNAUTHORIZED + ); + let flow = login(&provider, &api.url).await; + let transaction = begin(store, TransactionMode::Write).await.expect("expiry"); + let expired = transaction.now - chrono::Duration::minutes(10); + let (transaction, _) = query( + transaction, + "UPDATE sqb_oidc_login_flows SET expires_at=$1", + vec![Parameter::Time(expired)], + ) + .await + .expect("advance expiry"); + commit(transaction).await.expect("expiry commit"); + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::UNAUTHORIZED + ); + let machine = provider + .client + .get(format!( + "{}/api/locations/{}/health", + api.url, + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &flow.cookie) + .send() + .await + .expect("machine boundary"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + let transaction = begin(store, TransactionMode::Read).await.expect("evidence"); + let (transaction, rows) = query(transaction, + "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_password_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='oidc.authenticated') AS authentications", + vec![]).await.expect("no provisioning"); + assert_eq!( + json!(rows[0]), + json!({"principals":2,"sessions":0,"tokens":0,"authentications":test_case.expected_authentications}), + "{}", + test_case.description + ); + commit(transaction).await.expect("inspection commit"); + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + stop_provider(provider).await; + }); + } +} + +#[test] +fn given_oidc_login_attempts_when_alternating_replicas_then_the_peer_limit_is_shared() { + let test_cases = [ + OidcThrottleTestCase { + description: "SQLite shared peer limit", + backend: Backend::Sqlite, + expected_allowed: 20, + }, + OidcThrottleTestCase { + description: "PostgreSQL shared peer limit", + backend: Backend::Postgres, + expected_allowed: 20, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + migrate_store(&fixture.store).await.expect("migrate"); + let replica = second_store(&fixture.store).await; + let provider = start_provider().await; + let settings = OidcAuthentication { + origin: "https://control.example.com".into(), + issuer: provider.state.issuer.clone(), + client_id: "orders-control".into(), + client_secret_env: None, + certificate_authority: Some(provider.authority.clone()), + identities: BTreeMap::new(), + }; + let api = start_api_with_authentication( + &fixture.store, + ApiAuthentication { + oidc: Some(settings.clone()), + ..Default::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + oidc: Some(settings), + ..Default::default() + }, + ) + .await; + let mut statuses = Vec::new(); + for url in [&api.url, &other.url].into_iter().cycle().take(22) { + statuses.push( + provider + .client + .get(format!("{url}/api/auth/oidc/login")) + .header(header::HOST, "control.example.com") + .send() + .await + .expect("login attempt") + .status(), + ); + } + assert_eq!( + &statuses[..test_case.expected_allowed], + vec![StatusCode::SEE_OTHER; test_case.expected_allowed], + "{}", + test_case.description + ); + assert_eq!( + &statuses[test_case.expected_allowed..], + [StatusCode::TOO_MANY_REQUESTS; 2] + ); + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + stop_provider(provider).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 2a74905058..90355b36c0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -87,3 +87,15 @@ pub(crate) struct ProxyApiTestCase { pub(crate) trusted_peer: &'static str, pub(crate) expected_authentications: usize, } + +pub(crate) struct OidcApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_authentications: usize, +} + +pub(crate) struct OidcThrottleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_allowed: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index b7f7d2bafd..203aebd551 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -23,6 +23,7 @@ pub(crate) async fn fingerprint_history( // Upgrade the immediate predecessor while published snapshots and freshness facts exist. let mut transaction = begin(store, TransactionMode::Write).await.expect("upgrade"); for table in [ + "sqb_oidc_login_flows", "sqb_authentication_revision", "sqb_password_sessions", "sqb_password_login_attempts", @@ -45,7 +46,7 @@ pub(crate) async fn fingerprint_history( .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs index dd888e0ffd..3aedf94789 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs @@ -8,11 +8,12 @@ use uuid::Uuid; use crate::credentials::main::{ authenticate_session::authenticate_session, authenticate_token::authenticate_token, create_principal::create_principal, login_password::login_password, - reserve_password_attempt::reserve_password_attempt, revoke_session::revoke_session, + reserve_authentication_attempt::reserve_authentication_attempt, revoke_session::revoke_session, set_password::set_password, verify_password::verify_password, }; use crate::credentials::models::{ - AuditContext, PasswordAttempt, PasswordAuthority, PasswordLogin, PrincipalKind, SessionAccess, + AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, PrincipalKind, + SessionAccess, }; use crate::credentials::tests::test_sessions::test_types::SessionTestCase; use crate::store::main::{ @@ -148,10 +149,10 @@ fn given_password_sessions_when_replicas_race_or_auditing_fails_then_throttles_a let name = format!("customers-{index}"); let context = &another_peer; async move { - reserve_password_attempt( + reserve_authentication_attempt( stores[index % 2], context, - PasswordAttempt::Login(&name), + AuthenticationAttempt::Login(&name), ) .await .expect("shared peer reservation") diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 9eec7fbb3e..cfb8c1e19f 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -426,6 +426,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .await .expect("upgrade fixture"); for table in [ + "sqb_oidc_login_flows", "sqb_authentication_revision", "sqb_password_sessions", "sqb_password_login_attempts", diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 96b8b3d1b1..526a406cd9 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_authentication_revision; \ + "DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_decisive_history; \ From 7e19995a8939b8f0cbede2f4ad5112bce6a69d21 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:24:33 +0000 Subject: [PATCH 076/138] feat: package and verify redeploy-safe Compose orchestration --- .dockerignore | 11 + .github/workflows/verify.yml | 38 ++- .gitignore | 1 + Makefile | 16 ++ .../orchestration/container_deployment.json | 5 + compose.yml | 82 ++++++ .../src/native_commands/_helpers/execution.rs | 17 +- .../src/native_commands/main/_run_control.rs | 4 +- .../src/workers/_helpers/docker.rs | 2 +- .../src/workers/main/docker_image_sources.rs | 19 ++ crates/sqlbuild-cli/src/workers/main/mod.rs | 1 + .../api/_helpers/resources/docker_image.rs | 74 ++++++ .../src/api/_helpers/resources/mod.rs | 1 + .../src/api/_helpers/resources/snapshots.rs | 9 +- .../src/api/constants.rs | 2 + .../src/api/main/serve_api.rs | 5 +- .../sqlbuild-orchestration/src/api/models.rs | 9 + .../tests/api/helpers.rs | 2 +- deploy/Dockerfile | 27 ++ deploy/README.md | 147 +++++++++++ deploy/compose.dev.yml | 5 + deploy/compose.env.example | 8 + deploy/project/models/orders.sql | 3 + deploy/project/orchestration/orders.py | 10 + deploy/project/sqlbuild_project.toml | 8 + .../publication/_test_types.py | 6 + .../publication/docker/compose/helpers.py | 238 ++++++++++++++++++ .../publication/test_compose.py | 228 +++++++++++++++++ 28 files changed, 962 insertions(+), 16 deletions(-) create mode 100644 .dockerignore create mode 100644 benchmarks/duckshop/edge_cases/orchestration/container_deployment.json create mode 100644 compose.yml create mode 100644 crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs create mode 100644 deploy/Dockerfile create mode 100644 deploy/README.md create mode 100644 deploy/compose.dev.yml create mode 100644 deploy/compose.env.example create mode 100644 deploy/project/models/orders.sql create mode 100644 deploy/project/orchestration/orders.py create mode 100644 deploy/project/sqlbuild_project.toml create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..c7b7e36fac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +** +!dist/ +!dist/*.whl +!deploy/ +!deploy/Dockerfile +!deploy/project/ +!deploy/project/sqlbuild_project.toml +!deploy/project/models/ +!deploy/project/models/*.sql +!deploy/project/orchestration/ +!deploy/project/orchestration/*.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 410ad4f6c5..3b35960b21 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -142,7 +142,7 @@ jobs: *) docs_only=false ;; esac case "${path}" in - crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml) + crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json) wheels=true ;; esac @@ -982,6 +982,33 @@ jobs: uses: ./.github/workflows/build-distributions.yml with: sdist: false + upload: true + retention-days: 3 + + compose: + name: Packaged Compose lifecycle + needs: [changes, lockfile, wheels] + if: needs.changes.outputs.wheels == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + - name: Install Python test dependencies without rebuilding the packages + run: | + uv python install 3.12 + uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - uses: actions/download-artifact@v4 + with: + name: dist-wheel-linux-x86_64 + path: dist + - name: Install the same wheels used by the images + run: uv pip install --no-deps dist/*.whl + - name: Verify packaged Compose deployment and operational recovery + run: make test-compose-packaged verify: # The required Verify of a Release Please pull request is the commit status that the trusted @@ -1015,6 +1042,7 @@ jobs: postgres, clickhouse, wheels, + compose, ] runs-on: ubuntu-24.04 steps: @@ -1038,6 +1066,7 @@ jobs: POSTGRES_RESULT: ${{ needs.postgres.result }} CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }} WHEELS_RESULT: ${{ needs.wheels.result }} + COMPOSE_RESULT: ${{ needs.compose.result }} run: | if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then echo "Required classification or lockfile checks did not pass" >&2 @@ -1048,6 +1077,10 @@ jobs: echo "Required static checks did not pass: ${CHECKS_RESULT}" >&2 exit 1 fi + if [[ "${WHEELS_RESULT}" == "success" && "${COMPOSE_RESULT}" != "success" ]]; then + echo "Required packaged Compose verification did not pass: ${COMPOSE_RESULT}" >&2 + exit 1 + fi for result in \ "${E2E_SHARDS_RESULT}" \ "${COMPILER_DIFFERENTIAL_RESULT}" \ @@ -1062,7 +1095,8 @@ jobs: "${RULES_PERFORMANCE_RESULT}" \ "${POSTGRES_RESULT}" \ "${CLICKHOUSE_RESULT}" \ - "${WHEELS_RESULT}"; do + "${WHEELS_RESULT}" \ + "${COMPOSE_RESULT}"; do if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then echo "A selected verification suite did not pass: ${result}" >&2 exit 1 diff --git a/.gitignore b/.gitignore index 3bf9e0897f..249dfe1fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ target/ *.dylib .dlt/ .amp/portals/* +/.env # Local SQLBuild overrides sqlbuild_local.yml diff --git a/Makefile b/Makefile index 911e79832f..b795940c4c 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,22 @@ dev-native: fast-native: uv run --no-sync maturin develop --profile fast --skip-install +COMPOSE_BUILD_PROFILE ?= release +COMPOSE_BUILD_NETWORK ?= default +COMPOSE_PYTHON_VERSION ?= 3.12 + +.PHONY: test-compose +test-compose: + uv run --no-sync maturin build --profile $(COMPOSE_BUILD_PROFILE) --compatibility linux --out dist + uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --profile $(COMPOSE_BUILD_PROFILE) --compatibility linux --out dist + $(MAKE) test-compose-packaged + +.PHONY: test-compose-packaged +test-compose-packaged: + docker build --network $(COMPOSE_BUILD_NETWORK) --file deploy/Dockerfile --target server --build-arg PYTHON_VERSION=$(COMPOSE_PYTHON_VERSION) --tag sqlbuild-server:test . + docker build --network $(COMPOSE_BUILD_NETWORK) --file deploy/Dockerfile --target location --build-arg PYTHON_VERSION=$(COMPOSE_PYTHON_VERSION) --build-arg SQLBUILD_GIT_REVISION=$$(git rev-parse HEAD) --tag sqlbuild-location:test . + SQLBUILD_REQUIRE_COMPOSE_TESTS=1 uv run --no-sync pytest tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py -n auto --dist loadfile -vv + rust-check: rust-lint uv run fensu check $(MAKE) test-native diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_deployment.json b/benchmarks/duckshop/edge_cases/orchestration/container_deployment.json new file mode 100644 index 0000000000..d1f1a8783e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/container_deployment.json @@ -0,0 +1,5 @@ +{ + "first": "SELECT 37 AS order_id, sum(sin(value)) AS total FROM range(2000000000) t(value)", + "second": "SELECT 41 AS order_id, 0.0 AS total", + "broken": "SELECT FROM" +} diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000000..f9decab88a --- /dev/null +++ b/compose.yml @@ -0,0 +1,82 @@ +name: ${COMPOSE_PROJECT_NAME:-sqlbuild} + +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: sqlbuild + POSTGRES_DB: sqlbuild + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + healthcheck: + test: [CMD-SHELL, "pg_isready -U sqlbuild -d sqlbuild"] + interval: 2s + timeout: 2s + retries: 30 + volumes: + - control:/var/lib/postgresql/data + networks: [control] + + server: + image: ${SQLBUILD_SERVER_IMAGE:-sqlbuild-server:local} + build: + context: . + dockerfile: deploy/Dockerfile + target: server + args: + PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + environment: + - SQLBUILD_CONTROL_DATABASE_URL=${SQLBUILD_CONTROL_DATABASE_URL:?Set SQLBUILD_CONTROL_DATABASE_URL} + - SQLBUILD_PLATFORM_WEBHOOK_URL + group_add: ["${DOCKER_GID:?Set DOCKER_GID to the Docker socket group}"] + volumes: + - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock + configs: + - source: launchers + target: /etc/sqlbuild/launchers.json + ports: + - "${SQLBUILD_BIND_ADDRESS:-127.0.0.1}:${SQLBUILD_PORT:-8000}:8000" + healthcheck: + test: [CMD, curl, --fail, --silent, --output, /dev/null, "http://localhost:8000/api/openapi.json"] + interval: 2s + timeout: 2s + retries: 30 + depends_on: + postgres: + condition: service_healthy + networks: [control, workers] + + location: + image: ${SQLBUILD_LOCATION_IMAGE:-sqlbuild-orders:local} + container_name: ${COMPOSE_PROJECT_NAME:-sqlbuild}-location + build: + context: . + dockerfile: deploy/Dockerfile + target: location + args: + PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + SQLBUILD_GIT_REVISION: ${SQLBUILD_GIT_REVISION:?Set SQLBUILD_GIT_REVISION} + command: [location, serve, --server, "http://server:8000", --location-id, "${SQLBUILD_LOCATION_ID:?Set SQLBUILD_LOCATION_ID}", --target, development] + environment: + SQLBUILD_LOCATION_TOKEN: ${SQLBUILD_LOCATION_TOKEN:-} + volumes: + - warehouse:/warehouse + cpus: 2 + mem_limit: 1g + depends_on: + server: + condition: service_healthy + networks: [workers] + +configs: + launchers: + content: | + {"docker":[{"location_id":"${SQLBUILD_LOCATION_ID}","target":"development","server_url":"http://server:8000","max_workers":1,"socket":"/var/run/docker.sock","location_container":"${COMPOSE_PROJECT_NAME:-sqlbuild}-location","executable":"/usr/local/bin/sqb"}]} + +volumes: + control: + warehouse: + +networks: + control: + internal: true + workers: diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 9859011f57..b36264baaf 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -9,7 +9,7 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; -use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope}; +use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, DockerImageSource}; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; @@ -57,6 +57,7 @@ use crate::native_commands::models::{ LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; use crate::native_commands::types::LocationPublication; +use crate::workers::main::docker_image_sources::docker_image_sources; use crate::workers::main::open_development_workers::open_development_workers; use crate::workers::main::open_workers::open_workers; use crate::workers::main::terminate_child::terminate_child; @@ -86,10 +87,10 @@ pub(crate) async fn run_development_roles( tasks.spawn(serve_api( store.clone(), listener, - ApiAuthentication { + (ApiAuthentication { development_scope: Some(scope), ..ApiAuthentication::default() - }, + }, Vec::new()), (move || { drop(write_output(&[(Stream::Stderr, &format!( "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", @@ -495,13 +496,17 @@ pub(crate) async fn open_control_store( } /// Decode privileged CLI configuration before handing process ownership to the workers domain. -pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { - open_workers( +pub(crate) fn open_control_workers( + namespace: &Namespace, +) -> Result<(Workers, Vec), ControlError> { + let workers = open_workers( text(namespace, "control_launchers").as_deref(), text(namespace, "control_database_env") .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), !flag(namespace, "control_no_scheduler"), - ) + )?; + let images = docker_image_sources(&workers); + Ok((workers, images)) } pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 0be6102190..184649a83c 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -29,7 +29,7 @@ pub(crate) async fn run_control( let address = control_address(namespace, standalone)?; let authentication = control_authentication(namespace)?; let store = open_control_store(namespace).await?; - let workers = open_control_workers(namespace)?; + let (workers, images) = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); let (_changes, changes) = watch::channel(()); let mut tasks = tokio::task::JoinSet::new(); @@ -43,7 +43,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - authentication, + (authentication, images), ( move || { drop(write_output(&[( diff --git a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs index 7b0d5c9633..746ddf0c8e 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs @@ -86,7 +86,7 @@ impl DockerWorker { "WorkingDir": project, "User": parent["Config"]["User"], "Env": variables, "Labels": {"io.sqlbuild.claim-id": self.claim_id.to_string()}, "HostConfig": { - "NetworkMode": format!("container:{}", location.location_container), + "NetworkMode": parent["HostConfig"]["NetworkMode"], "VolumesFrom": [location.location_container], "Memory": parent["HostConfig"]["Memory"], "MemorySwap": parent["HostConfig"]["MemorySwap"], "NanoCpus": parent["HostConfig"]["NanoCpus"], "PidsLimit": parent["HostConfig"]["PidsLimit"], diff --git a/crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs b/crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs new file mode 100644 index 0000000000..09bba564fc --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs @@ -0,0 +1,19 @@ +//! Derive publication image sources from the same privileged Docker launcher configuration. + +use sqlbuild_orchestration::api::models::DockerImageSource; + +use crate::workers::models::Workers; + +pub(crate) fn docker_image_sources(workers: &Workers) -> Vec { + workers + .configuration + .docker + .iter() + .map(|location| DockerImageSource { + location_id: location.location_id, + target: location.target.clone(), + socket: location.socket.clone(), + container: location.location_container.clone(), + }) + .collect() +} diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs index def6c0f550..576cfbff61 100644 --- a/crates/sqlbuild-cli/src/workers/main/mod.rs +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod docker_image_sources; pub(crate) mod open_development_workers; pub(crate) mod open_workers; pub(crate) mod terminate_child; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs new file mode 100644 index 0000000000..b9d42619cd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs @@ -0,0 +1,74 @@ +//! Pin Docker publications to physical image evidence held by the privileged server. + +use axum::http::StatusCode; + +#[cfg(unix)] +use crate::api::constants::DOCKER_IMAGE_HEX_LENGTH; +use crate::api::errors::ApiError; +use crate::api::models::DockerImageSource; +use crate::snapshots::models::SnapshotInput; + +pub(crate) async fn pin( + sources: &[DockerImageSource], + mut input: SnapshotInput, +) -> Result { + let Some(source) = sources + .iter() + .find(|source| source.location_id == input.location_id && source.target == input.target) + else { + return Ok(input); + }; + let image = inspect(source).await?; + if input + .image_id + .as_ref() + .is_some_and(|claimed| *claimed != image) + { + return Err(ApiError { + status: StatusCode::CONFLICT, + message: "published image differs from the configured Docker location image".into(), + }); + } + input.image_id = Some(image); + Ok(input) +} + +#[cfg(unix)] +async fn inspect(source: &DockerImageSource) -> Result { + let client = reqwest::Client::builder() + .no_proxy() + .unix_socket(source.socket.as_path()) + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(|_| unavailable())?; + let response: serde_json::Value = client + .get(format!( + "http://localhost/containers/{}/json", + source.container + )) + .send() + .await + .map_err(|_| unavailable())? + .error_for_status() + .map_err(|_| unavailable())? + .json() + .await + .map_err(|_| unavailable())?; + let image = response["Image"].as_str().ok_or_else(unavailable)?; + let digest = image.strip_prefix("sha256:").ok_or_else(unavailable)?; + (digest.len() == DOCKER_IMAGE_HEX_LENGTH && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then(|| image.to_owned()) + .ok_or_else(unavailable) +} + +#[cfg(not(unix))] +async fn inspect(_source: &DockerImageSource) -> Result { + Err(unavailable()) +} + +fn unavailable() -> ApiError { + ApiError { + status: StatusCode::SERVICE_UNAVAILABLE, + message: "cannot inspect the configured Docker location image".into(), + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 71e23cbbb7..d241beee10 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod changes; pub(crate) mod client; +pub(crate) mod docker_image; pub(crate) mod location_health; pub(crate) mod location_work; pub(crate) mod router; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs index 7e54518ab7..bf53a25638 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs @@ -6,8 +6,9 @@ use axum::{Extension, Json}; use uuid::Uuid; use crate::api::_helpers::authentication::bearer::require_location; +use crate::api::_helpers::resources::docker_image; use crate::api::errors::ApiError; -use crate::api::models::DevelopmentScope; +use crate::api::models::{DevelopmentScope, DockerImageSource}; use crate::credentials::models::TokenIdentity; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::current_snapshot_record::current_snapshot_record; @@ -24,7 +25,10 @@ use crate::store::models::ControlStore; (status = 409, description = "Stale generation")), security(("bearer" = [])))] pub(crate) async fn publish( State(store): State, - Extension(identity): Extension, + (Extension(identity), Extension(images)): ( + Extension, + Extension>>, + ), (Path((location_id, target)), development): ( Path<(Uuid, String)>, Option>, @@ -49,6 +53,7 @@ pub(crate) async fn publish( } else { false }; + let input = docker_image::pin(&images, input).await?; Ok(Json(if pause_new { publish_development_snapshot(&store, &input).await? } else { diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index b4f810e892..c5c94efdac 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -9,5 +9,7 @@ pub(crate) const PASSWORD_CSRF_HEADER: &str = "x-sqb-csrf"; pub(crate) const PROXY_USER_HEADER: &str = "x-sqb-user"; pub(crate) const OIDC_FLOW_COOKIE: &str = "__Host-sqb-oidc"; pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024; +#[cfg(unix)] +pub(crate) const DOCKER_IMAGE_HEX_LENGTH: usize = 64; pub(crate) const HTTPS_SCHEME: &str = "https"; pub(crate) const ROOT_PATH: &str = "/"; diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 7bc810083d..4d0e9ec477 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -6,7 +6,7 @@ use axum::{Extension, Json, Router, middleware}; use crate::api::_helpers::authentication::{bearer::local_request, configure::configure}; use crate::api::_helpers::resources::{changes, router::router}; use crate::api::main::openapi_document::openapi_document; -use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope}; +use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, DockerImageSource}; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; @@ -15,7 +15,7 @@ use crate::store::models::ControlStore; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, - authentication: ApiAuthentication, + (authentication, images): (ApiAuthentication, Vec), (ready, shutdown): ( impl FnOnce() + Send + 'static, impl Future + Send + 'static, @@ -33,6 +33,7 @@ pub async fn serve_api( }; let application = router(&store, feed) .merge(authentication_routes) + .layer(Extension(std::sync::Arc::new(images))) .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 698a7aa568..621d56390f 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -109,6 +109,15 @@ pub struct ApiAuthentication { pub oidc: Option, } +/// Only the server's installation configuration selects the daemon and container to inspect. +#[derive(Clone, Debug)] +pub struct DockerImageSource { + pub location_id: uuid::Uuid, + pub target: String, + pub socket: std::path::PathBuf, + pub container: String, +} + #[derive(Clone, Debug)] pub(crate) struct BrowserOrigin(pub(crate) String); diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 4f5b42e081..63c313fca8 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -22,7 +22,7 @@ pub(crate) async fn start_api_with_authentication( let task = tokio::spawn(serve_api( store, listener, - authentication, + (authentication, Vec::new()), (|| {}, async { let _ = signal.await; }), diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000000..666ff222f7 --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,27 @@ +# Build wheels from this checkout before building these images; see deploy/README.md. +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime +COPY dist/*.whl /wheels/ +RUN python -m pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels \ + && useradd --uid 10001 --create-home sqlbuild \ + && mkdir /project /warehouse && chown sqlbuild:sqlbuild /project /warehouse +ENV PYTHONUNBUFFERED=1 +WORKDIR /project +USER 10001:10001 +ENTRYPOINT ["sqb"] + +FROM runtime AS location +ARG SQLBUILD_GIT_REVISION +RUN test -n "$SQLBUILD_GIT_REVISION" +ENV SQLBUILD_GIT_REVISION=${SQLBUILD_GIT_REVISION} +COPY --chown=10001:10001 deploy/project/ /project/ + +# The control plane has no Python interpreter, project, adapter, or warehouse credentials. +FROM debian:bookworm-slim AS server +RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --uid 10001 --create-home sqlbuild +COPY --from=runtime /usr/local/bin/sqb /usr/local/bin/sqb +USER 10001:10001 +ENTRYPOINT ["sqb"] +CMD ["server", "--host", "0.0.0.0", "--port", "8000", "--launchers", "/etc/sqlbuild/launchers.json"] diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000000..a0d2ceaab0 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,147 @@ +# Run the reference Docker Compose stack + +This Linux reference runs PostgreSQL, the native SQLBuild server/scheduler, and one outbound +Python code location. Workers use Docker only. The server image contains no Python or project. +The location image includes the example orders project and its Python dependencies. + +Use Docker Engine with a Unix socket and Docker Compose 2.23.1 or newer, Python 3.12, `uv`, +a Rust toolchain, and `jq`. Build and test from a trusted checkout. Docker socket access is +host-administrator authority: only the server gets that socket and control-store credentials. +Locations and workers get warehouse access, but no control-store credentials or inbound ports. +The default server port binds to loopback, not the public network. + +## Build the images + +Build matching wheels from this checkout before building the images: + +```sh +uv sync --locked --all-extras +uv run --no-sync maturin build --release --compatibility linux --out dist +uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --release --compatibility linux --out dist +cp deploy/compose.env.example .env +``` + +Set the ignored `.env` values. Use a unique PostgreSQL password and put its URL-encoded value +in `SQLBUILD_CONTROL_DATABASE_URL`, for example +`postgresql://sqlbuild:@postgres:5432/sqlbuild`. +Set `DOCKER_GID` to `stat -c %g /var/run/docker.sock` and `SQLBUILD_GIT_REVISION` to +`git rev-parse HEAD`. Keep only the matching SQLBuild and CLI wheels in `dist/`; their Python +version and architecture must match the image's `PYTHON_VERSION` build argument and Docker host. +The default image uses Python 3.12. Do not put credentials in build arguments or project files. +The Docker build context includes only wheels and the example project. + +```sh +docker compose build +docker compose up -d --wait postgres +``` + +## Register and start the code location + +Run these commands on the trusted control-store owner's host. They use the server image's +direct-store administration path, not an HTTP or person-role grant: + +```sh +docker compose run --rm --no-deps server admin create-location orders +``` + +Replace `.env`'s placeholder `SQLBUILD_LOCATION_ID` with the returned `location_id`. Issue a +bounded credential, using a future RFC 3339 expiry you choose: + +```sh +docker compose run --rm --no-deps server admin issue-location-token "$SQLBUILD_LOCATION_ID" --expires-at "$EXPIRY" +``` + +Shell variables above must be set explicitly; Compose's `.env` file is not sourced by the shell. +Store the one-time `bearer` in `.env` as `SQLBUILD_LOCATION_TOKEN`, restrict the file to its owner +(`chmod 600 .env`), and start the stack: + +```sh +docker compose up -d --wait server location +docker compose logs -f server location +``` + +The location watches and compiles its baked project, publishes a successful snapshot, and checks +its warehouse connection. The example schedule runs every minute. The server resolves the +running container's actual Docker image ID at publication; no `--image-id` or Docker socket is +needed in the location. Workers recompile in that pinned image and verify environment and node +hashes before writing. The warehouse persists in its named volume. + +For your project, build the `runtime` target as a base, then extend it with `COPY --chown=10001:10001` +of your project to `/project`, install its locked dependencies, and set `SQLBUILD_GIT_REVISION`. +Set `SQLBUILD_LOCATION_IMAGE` to that project image. Give location UID 10001 access to its +warehouse volumes. Never mount control credentials, location-token files, or Docker sockets +into the location: workers inherit its mounts and filtered warehouse environment. + +## Redeploy without interrupting a run + +Build the new project into a new revision-tagged image. Set `SQLBUILD_LOCATION_IMAGE` to that +image and update only the location: + +```sh +docker compose up -d --no-deps location +``` + +Active workers keep their existing image, process, mounts, and independent network endpoint. +New publications affect only new runs. A failed compile does not replace the last good snapshot; +its scheduled jobs still launch from the last good image even while the new location is broken. +Keep the warehouse mounts, network, working directory, and warehouse environment compatible with +that image. Changing those deployment settings requires a coordinated maintenance window. + +Retain every image referenced by a published snapshot or run. Do not run indiscriminate image +pruning or delete those images. Inspect `sqb_snapshots.image_id` and `sqb_runs.image_id` through +the trusted control-store owner before implementing retention. Removing a required image makes +a queued or historical run unlaunchable. + +For bind-mounted source development, explicitly add the development overlay: + +```sh +docker compose -f compose.yml -f deploy/compose.dev.yml up -d +``` + +The location watcher republishes changed source. Workers reject source drift against a pinned +snapshot; the bind mount is not an immutable deployment and does not provide old-source fallback. + +## Credentials, identity, and platform events + +Rotate a location credential with `docker compose run --rm --no-deps server admin rotate-token +TOKEN_UUID`, store its one-time replacement bearer, and recreate the location. The scope and +expiry stay unchanged, and the old credential is revoked atomically. Service-account principal +tokens are separate from location/run tokens. Person identities do not become machine bearers. + +For opt-in OIDC identity verification, mount a server-owned JSON file with `origin`, `issuer`, +`client_id`, and explicit `identities` subject-to-existing-person UUID bindings; append +`--oidc-config /etc/sqlbuild/oidc.json` to the server command. Both origin and issuer require +HTTPS. Terminate HTTPS at a trusted proxy, preserve the external Host, restrict backend access, +and register `/api/auth/oidc/callback` at the issuer. This verifies an explicitly bound enabled +person; it does not provision people, issue a browser session, or grant roles. The reference +does not assign an initial administrator or authorize non-admin operations. + +Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in `.env` before creating the server to receive the native +platform stream. Leave it unset to disable delivery. Receivers must deduplicate by `event_id` +or the matching `Idempotency-Key` header: delivery is at least once and retries after failure. +The native watchdog runs independently of location Python and scheduler leadership. It observes +heartbeats, overdue ticks, missing leadership, and locations silent for over 90 seconds; its +30-second scan and delivery polling add detection latency. Monitor platform heartbeats from +outside this stack too: a stopped server cannot report its own outage. + +## Back up and upgrade + +Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Also +back up warehouse data, the restricted `.env`/identity configuration, and referenced project +images. Keep backup credentials and output outside the repository. Test restoration before an +upgrade. Ordinary `docker compose down` preserves named volumes; `down --volumes` destroys them. + +Upgrade matching server/location wheels and images together, retaining referenced old project +images. The native server applies additive control-store migrations before its API reports +ready. Do not start an older server against a newer schema without a documented compatible +rollback or restored backup. Scheduler leadership resumes from durable ticks after restart; +catch-up follows each declaration's `latest` or bounded `all` policy, not a fresh schedule. +The Docker launcher adopts an existing claim-labeled worker rather than starting a duplicate. + +## Verify the packaged lifecycle locally + +`make test-compose` builds matching wheels and images, then tests scheduled Docker execution, +redeploy during an active run, failed-deploy fallback, scheduler catch-up, credential/mount +isolation, and the stopped-location platform webhook against PostgreSQL. It creates disposable +containers and volumes and removes them on exit. It needs Docker, Compose, and network access +for image and Python dependency downloads; it never deploys to an external installation. diff --git a/deploy/compose.dev.yml b/deploy/compose.dev.yml new file mode 100644 index 0000000000..f18be2221b --- /dev/null +++ b/deploy/compose.dev.yml @@ -0,0 +1,5 @@ +# Opt in with: docker compose -f compose.yml -f deploy/compose.dev.yml up -d +services: + location: + volumes: + - ./deploy/project:/project diff --git a/deploy/compose.env.example b/deploy/compose.env.example new file mode 100644 index 0000000000..a667e912e0 --- /dev/null +++ b/deploy/compose.env.example @@ -0,0 +1,8 @@ +# Copy to the ignored root .env; set the password and matching URL before starting PostgreSQL. +POSTGRES_PASSWORD= +SQLBUILD_CONTROL_DATABASE_URL= +DOCKER_GID= +SQLBUILD_GIT_REVISION= +# Replace this placeholder with create-location's assigned identity before starting the location. +SQLBUILD_LOCATION_ID=00000000-0000-0000-0000-000000000001 +SQLBUILD_LOCATION_TOKEN= diff --git a/deploy/project/models/orders.sql b/deploy/project/models/orders.sql new file mode 100644 index 0000000000..3c8bb14e45 --- /dev/null +++ b/deploy/project/models/orders.sql @@ -0,0 +1,3 @@ +MODEL (description 'Example orders.', materialized table); + +SELECT 37 AS order_id diff --git a/deploy/project/orchestration/orders.py b/deploy/project/orchestration/orders.py new file mode 100644 index 0000000000..47aeb4c1f3 --- /dev/null +++ b/deploy/project/orchestration/orders.py @@ -0,0 +1,10 @@ +"""A small scheduled project baked into the reference location image.""" + +from sqlbuild.orchestration import job, schedule + +orders = job( + "orders", + select="orders", + description="Refresh orders every minute.", + schedule=schedule(cron="* * * * *", timezone="UTC"), +) diff --git a/deploy/project/sqlbuild_project.toml b/deploy/project/sqlbuild_project.toml new file mode 100644 index 0000000000..7a45950c85 --- /dev/null +++ b/deploy/project/sqlbuild_project.toml @@ -0,0 +1,8 @@ +name = "orders" +adapter = "duckdb" + +[connections.local] +database = "/warehouse/orders.duckdb" + +[targets.development] +schema = "main" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 2dfad6646e..1c7fae7b54 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -184,3 +184,9 @@ class DevelopmentTestCase: control_path: str expected_rows: list[tuple[int, int]] local_config: str = "" + + +@dataclass(frozen=True) +class ComposeDeploymentTestCase: + description: str + expected_warehouse_rows: list[list[int]] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py new file mode 100644 index 0000000000..00e948a51d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py @@ -0,0 +1,238 @@ +"""Own a disposable packaged PostgreSQL/HTTP/Docker stack without logging credentials.""" + +from __future__ import annotations + +import json +import os +import secrets +import shutil +import subprocess +import time +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from typing import Any +from uuid import uuid4 + + +class ComposeStack: + """One test owns its containers, images, PostgreSQL volume, and warehouse volume.""" + + def __init__(self, directory: Path) -> None: + self.directory = directory + self.name = f"sqlbuild-compose-{uuid4().hex}" + self.location_id = "00000000-0000-0000-0000-000000000001" + password: str = secrets.token_hex(24) + self.environment = os.environ | { + "COMPOSE_PROJECT_NAME": self.name, + "SQLBUILD_PORT": "0", + "DOCKER_GID": str(Path("/var/run/docker.sock").stat().st_gid), + "POSTGRES_PASSWORD": password, + "SQLBUILD_CONTROL_DATABASE_URL": f"postgresql://sqlbuild:{password}@postgres:5432/sqlbuild", + "SQLBUILD_SERVER_IMAGE": os.environ.get( + "SQLBUILD_TEST_SERVER_IMAGE", "sqlbuild-server:test" + ), + "SQLBUILD_LOCATION_IMAGE": f"{self.name}:first", + "SQLBUILD_LOCATION_ID": self.location_id, + "SQLBUILD_GIT_REVISION": "compose-acceptance", + } + self.override = directory / "override.json" + self.override.write_text( + json.dumps( + {"services": {"server": {"extra_hosts": ["host.docker.internal:host-gateway"]}}} + ) + ) + + def docker( + self, *arguments: str, check: bool = True, timeout: float = 120 + ) -> subprocess.CompletedProcess[str]: + started: float = time.monotonic() + result: subprocess.CompletedProcess[str] = subprocess.run( + ["docker", *arguments], + env=self.environment, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + with (self.directory / "commands.jsonl").open("a") as log: + log.write( + json.dumps( + { + "args": arguments, + "exit": result.returncode, + "seconds": time.monotonic() - started, + } + ) + + "\n" + ) + assert not check or result.returncode == 0, result.stderr + return result + + def compose(self, *arguments: str, check: bool = True) -> str: + return self.docker( + "compose", + "--file", + str(Path("compose.yml").resolve()), + "--file", + str(self.override), + *arguments, + check=check, + ).stdout.strip() + + def query(self, sql: str) -> list[dict[str, Any]]: + result: str = self.compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "sqlbuild", + "-d", + "sqlbuild", + "-At", + "-c", + f"SELECT COALESCE(json_agg(result),'[]'::json)::text FROM ({sql}) result", + ) + return json.loads(result) + + def wait(self, sql: str, *, timeout: float = 150) -> list[dict[str, Any]]: + deadline: float = time.monotonic() + timeout + rows: list[dict[str, Any]] = self.query(sql) + while not rows: + assert time.monotonic() < deadline, f"Timed out: {sql} (logs in {self.directory})" + time.sleep(0.5) + rows = self.query(sql) + with (self.directory / "observations.jsonl").open("a") as log: + log.write( + json.dumps({"at": datetime.now(UTC).isoformat(), "query": sql, "rows": rows}) + "\n" + ) + return rows + + def build_projects(self) -> list[str]: + cases: dict[str, str] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/container_deployment.json" + ).read_text() + ) + base: str = os.environ.get("SQLBUILD_TEST_LOCATION_IMAGE", "sqlbuild-location:test") + images: list[str] = [] + for version, sql in cases.items(): + project: Path = self.directory / version + (project / "models").mkdir(parents=True) + (project / "orchestration").mkdir() + for relative in ("sqlbuild_project.toml", "orchestration/orders.py"): + shutil.copyfile(Path("deploy/project") / relative, project / relative) + (project / "models/orders.sql").write_text( + "MODEL (description 'Example orders.', materialized table);\n" + sql + "\n" + ) + # Only source enters this build context; credentials stay in the parent process. + (project / "Dockerfile").write_text( + f"FROM {base}\nENV SQLBUILD_GIT_REVISION={version}\n" + "COPY --chown=10001:10001 . /project/\n" + ) + self.docker("build", "--tag", f"{self.name}:{version}", str(project)) + images.append( + self.docker( + "image", "inspect", "--format", "{{.Id}}", f"{self.name}:{version}" + ).stdout.strip() + ) + return images + + def register(self) -> None: + self.compose("up", "-d", "--wait", "postgres") + location: dict[str, Any] = json.loads( + self.compose("run", "--rm", "--no-deps", "server", "admin", "create-location", "orders") + ) + self.location_id = location["location_id"] + self.environment["SQLBUILD_LOCATION_ID"] = self.location_id + expiry: str = (datetime.now(UTC) + timedelta(hours=1)).isoformat() + credential: dict[str, Any] = json.loads( + self.compose( + "run", + "--rm", + "--no-deps", + "server", + "admin", + "issue-location-token", + self.location_id, + "--expires-at", + expiry, + ) + ) + self.environment["SQLBUILD_LOCATION_TOKEN"] = credential["bearer"] + + def redeploy(self, version: str) -> None: + self.environment["SQLBUILD_LOCATION_IMAGE"] = f"{self.name}:{version}" + self.compose("up", "-d", "--no-deps", "location") + + def close(self) -> None: + self.compose("stop", "--timeout", "5", "server", "location", check=False) + for service in ("server", "location", "postgres"): + (self.directory / f"{service}.log").write_text( + self.compose("logs", "--no-color", service, check=False) + ) + workers: list[str] = self.docker( + "ps", "--all", "--quiet", "--filter", f"name=sqlbuild-{self.location_id}-" + ).stdout.split() + for worker in workers: + (self.directory / f"worker-{worker}.log").write_text( + self.docker("logs", worker, check=False).stdout + ) + self.docker("rm", "--force", worker) + self.compose("down", "--volumes", "--remove-orphans") + for version in ("first", "second", "broken"): + self.docker("image", "rm", f"{self.name}:{version}", check=False) + + +@contextmanager +def platform_receiver() -> Iterator[tuple[int, list[dict[str, Any]]]]: + """Own the real webhook HTTP boundary even if stack cleanup itself fails.""" + events: list[dict[str, Any]] = [] + + class Receiver(BaseHTTPRequestHandler): + def do_POST(self) -> None: + event: dict[str, Any] = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + assert self.headers["Idempotency-Key"] == event["event_id"] + events.append(event) + self.send_response(204) + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + """Keep HTTP chatter out of test output.""" + + with ThreadingHTTPServer(("0.0.0.0", 0), Receiver) as receiver: + thread: Thread = Thread(target=receiver.serve_forever, daemon=True) + thread.start() + try: + yield receiver.server_port, events + finally: + receiver.shutdown() + thread.join(timeout=5) + assert not thread.is_alive() + + +def wait_for_location_outage( + events: list[dict[str, Any]], prior_events: set[str] +) -> dict[str, Any]: + """Wait for fresh delivery, not an outage already received during a prior restart.""" + deadline: float = time.monotonic() + 135 + matches: list[dict[str, Any]] = [] + while not matches: + assert time.monotonic() < deadline, ( + "Native platform sink did not report the stopped location" + ) + time.sleep(0.5) + matches = list( + filter( + lambda event: ( + event["event_id"] not in prior_events + and event["payload"]["platform"]["kind"] == "location_broken" + ), + events, + ) + ) + return matches[0] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py new file mode 100644 index 0000000000..e81b297865 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py @@ -0,0 +1,228 @@ +"""Acceptance at the distributed package boundary, not the source-tree runtime mounts.""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ComposeDeploymentTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.compose.helpers import ( + ComposeStack, + platform_receiver, + wait_for_location_outage, +) + + +@pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_COMPOSE_TESTS") != "1", + reason="Run make test-compose to build and test the packaged deployment", +) +@pytest.mark.parametrize( + "test_case", + [ + ComposeDeploymentTestCase( + "PostgreSQL control store and immutable DuckDB project images", [[41]] + ) + ], + ids=lambda case: case.description, +) +def test_given_packaged_services_when_redeployed_and_restarted_then_pinned_runs_and_platform_events_survive( + tmp_path: Path, + test_case: ComposeDeploymentTestCase, +) -> None: + stack: ComposeStack = ComposeStack(tmp_path) + with platform_receiver() as (port, events): + stack.environment["SQLBUILD_PLATFORM_WEBHOOK_URL"] = ( + f"http://host.docker.internal:{port}/events" + ) + try: + first_image, second_image, broken_image = stack.build_projects() + stack.register() + stack.compose("up", "-d", "--wait", "server") + stack.compose("up", "-d", "--no-deps", "location") + first: dict[str, Any] = stack.wait( + "SELECT * FROM sqb_runs WHERE status='running' AND origin='schedule'" + )[0] + assert first["image_id"] == first_image + worker = f"sqlbuild-{stack.location_id}-{first['run_id']}" + original: dict[str, Any] = json.loads( + stack.docker("inspect", "--format", "{{json .State}}", worker).stdout + ) + + # A real replacement removes the old network endpoint; merely restarting is insufficient. + stack.compose("stop", "--timeout", "1", "location") + stack.redeploy("second") + after: dict[str, Any] = json.loads( + stack.docker("inspect", "--format", "{{json .State}}", worker).stdout + ) + assert after["Running"] is True + assert after["Pid"] == original["Pid"] + assert ( + stack.docker("inspect", "--format", "{{.Image}}", worker).stdout.strip() + == first_image + ) + second: dict[str, Any] = stack.wait( + f"SELECT snapshot_id,image_id FROM sqb_snapshots WHERE image_id='{second_image}'" + )[0] + assert second["snapshot_id"] != first["snapshot_id"] + stack.wait( + f"SELECT run_id,status FROM sqb_runs WHERE run_id='{first['run_id']}' AND status='success'" + ) + + # These are real Docker/credential/mount boundaries, without exposing their values. + names: list[str] = json.loads( + stack.docker("inspect", "--format", "{{json .Config.Env}}", worker).stdout + ) + names = [entry.split("=", 1)[0] for entry in names] + assert "SQLBUILD_RUN_TOKEN" in names + assert not { + "SQLBUILD_LOCATION_TOKEN", + "SQLBUILD_CONTROL_DATABASE_URL", + "SQLBUILD_PLATFORM_WEBHOOK_URL", + }.intersection(names) + mounts: list[dict[str, Any]] = json.loads( + stack.docker("inspect", "--format", "{{json .Mounts}}", worker).stdout + ) + assert {mount["Destination"] for mount in mounts} == {"/warehouse"} + assert ( + stack.docker( + "inspect", "--format", "{{.HostConfig.NetworkMode}}", worker + ).stdout.strip() + == f"{stack.name}_workers" + ) + assert ( + stack.compose( + "exec", + "-T", + "server", + "sh", + "-c", + "! command -v python && ! command -v python3", + ) + == "" + ) + + # A location cannot substitute an operator-provided equivalent hash for physical evidence. + mismatch: subprocess.CompletedProcess[str] = stack.docker( + "exec", + f"{stack.name}-location", + "sqb", + "location", + "publish", + "--server", + "http://server:8000", + "--location-id", + stack.location_id, + "--target", + "development", + "--generation", + "99", + "--image-id", + "sha256:" + "0" * 64, + check=False, + ) + assert mismatch.returncode != 0 + assert "409" in mismatch.stderr + assert ( + stack.query("SELECT MAX(generation) AS generation FROM sqb_snapshots")[0][ + "generation" + ] + == 2 + ) + + stack.redeploy("broken") + failure: dict[str, Any] = stack.wait( + "SELECT occurred_at FROM recall_events WHERE event_type='location_health' AND payload->'report'->'health'->>'kind'='compile_failed' ORDER BY storage_id DESC" + )[0] + current: dict[str, Any] = stack.query( + "SELECT image_id,snapshot_id FROM sqb_snapshots JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + )[0] + assert current == second + assert current["image_id"] != broken_image + + # Queue the next scheduled occurrence from the surviving declaration, not a manual run. + fallback: dict[str, Any] = stack.wait( + f"SELECT run_id,status,image_id,snapshot_id FROM sqb_runs WHERE image_id='{second_image}' AND origin='schedule' AND queued_at>'{failure['occurred_at']}' AND status='success'" + )[0] + assert fallback["snapshot_id"] == second["snapshot_id"] + warehouse: str = stack.compose( + "exec", + "-T", + "location", + "python", + "-c", + "import duckdb,json; print(json.dumps(duckdb.connect('/warehouse/orders.duckdb',read_only=True).sql('SELECT order_id FROM orders').fetchall()))", + ) + assert json.loads(warehouse) == test_case.expected_warehouse_rows + + # Let a real minute boundary pass with no scheduler process, then catch up once. + stack.compose("stop", "--timeout", "5", "server") + before: int = stack.query("SELECT count(*) AS count FROM sqb_runs")[0]["count"] + due: str = stack.query("SELECT next_due_at FROM sqb_trigger_state")[0]["next_due_at"] + deadline: float = datetime.fromisoformat(due).timestamp() + 2 + while datetime.now(UTC).timestamp() < deadline: + time.sleep(0.5) + assert stack.query("SELECT count(*) AS count FROM sqb_runs")[0]["count"] == before + stack.compose("up", "-d", "--wait", "server") + stack.wait( + f"SELECT r.run_id,r.status,t.scheduled_for FROM sqb_runs r JOIN sqb_ticks t USING(tick_id) WHERE t.scheduled_for='{due}' AND r.status='success'" + ) + assert ( + stack.query( + "SELECT trigger_id,scheduled_for,count(*) FROM sqb_ticks GROUP BY trigger_id,scheduled_for HAVING count(*)>1" + ) + == [] + ) + + prior_events: set[str] = {event["event_id"] for event in events} + stack.compose("stop", "--timeout", "1", "location") + stopped_at: datetime = datetime.now(UTC) + broken: dict[str, Any] = wait_for_location_outage(events, prior_events) + assert broken["payload"]["platform"]["location_id"] == stack.location_id + assert broken["payload"]["platform"]["target"] == "development" + assert ( + datetime.fromisoformat(broken["payload"]["platform"]["last_seen_at"]) <= stopped_at + ) + + # Even a valid location bearer cannot publish without physical image evidence. + stack.compose("rm", "--force", "location") + unavailable: subprocess.CompletedProcess[str] = stack.docker( + "run", + "--rm", + "--network", + f"{stack.name}_workers", + "--env", + "SQLBUILD_LOCATION_TOKEN", + f"{stack.name}:second", + "location", + "publish", + "--server", + "http://server:8000", + "--location-id", + stack.location_id, + "--target", + "development", + "--generation", + "99", + check=False, + ) + assert unavailable.returncode != 0 + assert "503" in unavailable.stderr + assert ( + stack.query( + "SELECT image_id,snapshot_id FROM sqb_snapshots JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + )[0] + == second + ) + finally: + (tmp_path / "platform-events.json").write_text(json.dumps(events, indent=2)) + stack.close() From d505a6b74bae0401462fa4e8d85da5a25b8a356d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:30:52 +0000 Subject: [PATCH 077/138] fix: verify control-store restore and writable Compose development --- deploy/Dockerfile | 3 +- deploy/README.md | 18 ++-- deploy/compose.dev.yml | 6 ++ .../publication/docker/compose/helpers.py | 90 +++++++++++++++++-- .../publication/test_compose.py | 80 +++++++++++++++++ 5 files changed, 185 insertions(+), 12 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 666ff222f7..5ecf300a8a 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -4,7 +4,8 @@ FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime COPY dist/*.whl /wheels/ RUN python -m pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels \ && useradd --uid 10001 --create-home sqlbuild \ - && mkdir /project /warehouse && chown sqlbuild:sqlbuild /project /warehouse + && mkdir -p /project/target /project/logs /warehouse \ + && chown sqlbuild:sqlbuild /project /project/target /project/logs /warehouse ENV PYTHONUNBUFFERED=1 WORKDIR /project USER 10001:10001 diff --git a/deploy/README.md b/deploy/README.md index a0d2ceaab0..49fa889686 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -98,8 +98,11 @@ For bind-mounted source development, explicitly add the development overlay: docker compose -f compose.yml -f deploy/compose.dev.yml up -d ``` -The location watcher republishes changed source. Workers reject source drift against a pinned -snapshot; the bind mount is not an immutable deployment and does not provide old-source fallback. +The location watcher republishes changed source. The overlay keeps `target` and `logs` in +named volumes owned by the image's runtime user, so location and worker output does not require +changing the host source tree's ownership. Use the same overlay for `down --volumes` when removing +these development caches and logs. Workers reject source drift against a pinned snapshot; the +bind mount is not an immutable deployment and does not provide old-source fallback. ## Credentials, identity, and platform events @@ -142,6 +145,11 @@ The Docker launcher adopts an existing claim-labeled worker rather than starting `make test-compose` builds matching wheels and images, then tests scheduled Docker execution, redeploy during an active run, failed-deploy fallback, scheduler catch-up, credential/mount -isolation, and the stopped-location platform webhook against PostgreSQL. It creates disposable -containers and volumes and removes them on exit. It needs Docker, Compose, and network access -for image and Python dependency downloads; it never deploys to an external installation. +isolation, the development overlay's source reload and worker outputs, and the stopped-location +platform webhook against PostgreSQL. It also backs up the +populated control store, restores into an empty database, compares every public table's row count +and content digest before restarting writers, and checks that retained credentials and last-good +scheduling work after restore without duplicate ticks. This is a control-store restore check, +not an image, secret-file, or warehouse disaster-recovery test. It creates disposable containers +and volumes and removes them on exit. It needs Docker, Compose, and network access for image +and Python dependency downloads; it never deploys to an external installation. diff --git a/deploy/compose.dev.yml b/deploy/compose.dev.yml index f18be2221b..78c4d83f10 100644 --- a/deploy/compose.dev.yml +++ b/deploy/compose.dev.yml @@ -3,3 +3,9 @@ services: location: volumes: - ./deploy/project:/project + - development-cache:/project/target + - development-logs:/project/logs + +volumes: + development-cache: + development-logs: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py index 00e948a51d..74b846395e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py @@ -45,6 +45,7 @@ def __init__(self, directory: Path) -> None: {"services": {"server": {"extra_hosts": ["host.docker.internal:host-gateway"]}}} ) ) + self.compose_files = [Path("compose.yml").resolve(), self.override] def docker( self, *arguments: str, check: bool = True, timeout: float = 120 @@ -73,12 +74,12 @@ def docker( return result def compose(self, *arguments: str, check: bool = True) -> str: + configuration: list[str] = [] + for file in self.compose_files: + configuration.extend(("--file", str(file))) return self.docker( "compose", - "--file", - str(Path("compose.yml").resolve()), - "--file", - str(self.override), + *configuration, *arguments, check=check, ).stdout.strip() @@ -99,6 +100,82 @@ def query(self, sql: str) -> list[dict[str, Any]]: ) return json.loads(result) + def database_manifest(self) -> dict[str, list[dict[str, Any]]]: + """Compare persisted rows without exposing credential hashes or payloads in failures.""" + tables: list[dict[str, Any]] = self.query( + "SELECT tablename,quote_ident(tablename) AS identifier FROM pg_tables " + "WHERE schemaname='public' ORDER BY tablename" + ) + return { + table["tablename"]: self.query( + "SELECT count(*) AS rows,md5(COALESCE(string_agg(row_to_json(t)::text," + "E'\\n' ORDER BY row_to_json(t)::text),'')) AS digest " + f"FROM public.{table['identifier']} t" + ) + for table in tables + } + + def verify_control_store_restore(self, snapshot: dict[str, Any]) -> None: + """Restore the populated stack into an empty store before allowing either writer back.""" + self.compose("stop", "--timeout", "5", "server") + manifest: dict[str, list[dict[str, Any]]] = self.database_manifest() + assert manifest["sqb_runs"][0]["rows"] > 0 + assert manifest["sqb_ticks"][0]["rows"] > 0 + assert manifest["sqb_access_tokens"][0]["rows"] > 0 + self.compose( + "exec", + "-T", + "postgres", + "pg_dump", + "-U", + "sqlbuild", + "--format=custom", + "--file=/tmp/control.backup", + "sqlbuild", + ) + self.compose("exec", "-T", "postgres", "dropdb", "-U", "sqlbuild", "sqlbuild") + self.compose("exec", "-T", "postgres", "createdb", "-U", "sqlbuild", "sqlbuild") + assert self.database_manifest() == {} + self.compose( + "exec", + "-T", + "postgres", + "pg_restore", + "-U", + "sqlbuild", + "--exit-on-error", + "--dbname=sqlbuild", + "/tmp/control.backup", + ) + assert self.database_manifest() == manifest + (self.directory / "restore-manifest.json").write_text(json.dumps(manifest, indent=2)) + self.compose("exec", "-T", "postgres", "rm", "/tmp/control.backup") + + # Retained credentials authenticate the broken location without replacing last-good. + self.redeploy("broken") + resumed_at: str = datetime.now(UTC).isoformat() + self.compose("up", "-d", "--wait", "server") + self.wait( + "SELECT event_id FROM recall_events WHERE event_type='location_health' " + "AND payload->'report'->'health'->>'kind'='compile_failed' " + f"AND occurred_at>'{resumed_at}'" + ) + self.wait( + f"SELECT run_id FROM sqb_runs WHERE origin='schedule' AND queued_at>'{resumed_at}' " + f"AND snapshot_id='{snapshot['snapshot_id']}' AND status='success'" + ) + assert ( + self.query( + "SELECT trigger_id,scheduled_for,count(*) FROM sqb_ticks " + "GROUP BY trigger_id,scheduled_for HAVING count(*)>1" + ) + == [] + ) + assert self.query( + "SELECT image_id,snapshot_id FROM sqb_snapshots " + "JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + ) == [snapshot] + def wait(self, sql: str, *, timeout: float = 150) -> list[dict[str, Any]]: deadline: float = time.monotonic() + timeout rows: list[dict[str, Any]] = self.query(sql) @@ -179,9 +256,10 @@ def close(self) -> None: "ps", "--all", "--quiet", "--filter", f"name=sqlbuild-{self.location_id}-" ).stdout.split() for worker in workers: - (self.directory / f"worker-{worker}.log").write_text( - self.docker("logs", worker, check=False).stdout + output: subprocess.CompletedProcess[str] = self.docker( + "logs", "--timestamps", worker, check=False ) + (self.directory / f"worker-{worker}.log").write_text(output.stdout + output.stderr) self.docker("rm", "--force", worker) self.compose("down", "--volumes", "--remove-orphans") for version in ("first", "second", "broken"): diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py index e81b297865..30736d5b25 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py @@ -4,6 +4,7 @@ import json import os +import shutil import subprocess import time from datetime import UTC, datetime @@ -223,6 +224,85 @@ def test_given_packaged_services_when_redeployed_and_restarted_then_pinned_runs_ )[0] == second ) + + stack.verify_control_store_restore(second) finally: (tmp_path / "platform-events.json").write_text(json.dumps(events, indent=2)) stack.close() + + +@pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_COMPOSE_TESTS") != "1", + reason="Run make test-compose to build and test the packaged deployment", +) +@pytest.mark.parametrize( + "test_case", + [ + ComposeDeploymentTestCase( + "Host-owned development source with isolated runtime writes", [[41]] + ) + ], + ids=lambda case: case.description, +) +def test_given_host_owned_source_when_using_the_development_overlay_then_reloaded_workers_can_write_runtime_outputs( + tmp_path: Path, + test_case: ComposeDeploymentTestCase, +) -> None: + stack: ComposeStack = ComposeStack(tmp_path) + stack.environment["SQLBUILD_LOCATION_IMAGE"] = os.environ.get( + "SQLBUILD_TEST_LOCATION_IMAGE", "sqlbuild-location:test" + ) + project: Path = tmp_path / "source" + shutil.copytree(Path("deploy/project"), project) + assert project.stat().st_uid != 10001 + override: dict[str, Any] = json.loads(stack.override.read_text()) + override["services"]["location"] = { + "volumes": [{"type": "bind", "source": str(project), "target": "/project"}] + } + stack.override.write_text(json.dumps(override)) + stack.compose_files.insert(1, Path("deploy/compose.dev.yml").resolve()) + try: + stack.register() + stack.compose("up", "-d", "--wait", "server") + stack.compose("up", "-d", "--no-deps", "location") + first: dict[str, Any] = stack.wait( + "SELECT snapshot_id FROM sqb_snapshots WHERE generation=1", timeout=60 + )[0] + cases: dict[str, str] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/container_deployment.json" + ).read_text() + ) + (project / "models/orders.sql").write_text( + "MODEL (description 'Example orders.', materialized table);\n" + cases["second"] + "\n" + ) + second: dict[str, Any] = stack.wait( + "SELECT snapshot_id FROM sqb_snapshots WHERE generation=2", timeout=45 + )[0] + assert second != first + stack.wait( + f"SELECT run_id FROM sqb_runs WHERE snapshot_id='{second['snapshot_id']}' " + "AND origin='schedule' AND status='success'", + timeout=100, + ) + warehouse: str = stack.compose( + "exec", + "-T", + "location", + "python", + "-c", + "import duckdb,json; print(json.dumps(duckdb.connect('/warehouse/orders.duckdb',read_only=True).sql('SELECT order_id FROM orders').fetchall()))", + ) + assert json.loads(warehouse) == test_case.expected_warehouse_rows + (project / "models/orders.sql").write_text(cases["broken"] + "\n") + stack.wait( + "SELECT event_id FROM recall_events WHERE event_type='location_health' " + "AND payload->'report'->'health'->>'kind'='compile_failed'", + timeout=45, + ) + assert stack.query( + "SELECT snapshot_id FROM sqb_snapshots " + "JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + ) == [second] + finally: + stack.close() From b17fb321cc43abfe98f135f1a4259d79abe958ad Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:38:33 +0000 Subject: [PATCH 078/138] feat: add permission-enforced person operations Initialize the first admin once and protect the last password-usable admin with live transactional authorization. Add scoped built-in role assignment, person job/trigger/history routes and real browser-to-DuckDB acceptance. Require history rewrite permission for effective frozen cursor settings and maintain the wide-sensor fixture lease without changing its time budget. --- .../administrator_lifecycle.json | 17 + clients/typescript/src/schema.d.ts | 312 ++++++++++++++ .../sqlbuild-cli/src/argv/command_table.json | 17 + .../src/native_commands/_helpers/decision.rs | 6 + .../src/native_commands/main/_run_admin.rs | 5 + .../src/native_commands/models.rs | 1 + .../native_commands/admin/test_passwords.rs | 74 +++- .../tests/native_commands/test_types.rs | 5 + .../authentication/password_sessions.rs | 20 + .../authentication/password_transport.rs | 7 + .../api/_helpers/resources/administrators.rs | 51 +++ .../src/api/_helpers/resources/mod.rs | 2 + .../_helpers/resources/person_operations.rs | 70 ++++ .../sqlbuild-orchestration/src/api/errors.rs | 7 + .../sqlbuild-orchestration/src/api/models.rs | 9 +- .../_helpers/administrator_changes.rs | 105 +++++ .../_helpers/administrator_roles.rs | 115 ++++++ .../authorization/_helpers/administrators.rs | 79 ++++ .../src/authorization/_helpers/mod.rs | 3 + .../authorization/main/assign_builtin_role.rs | 27 ++ .../authorization/main/authorize_session.rs | 93 +++++ .../src/authorization/main/bootstrap_user.rs | 56 +++ .../main/change_administrator.rs | 33 ++ .../authorization/main/deny_authorization.rs | 37 ++ .../src/authorization/main/mod.rs | 5 + .../src/authorization/mod.rs | 4 + .../credentials/main/authenticate_session.rs | 36 +- .../main/authenticate_session_in.rs | 37 ++ .../src/credentials/main/create_principal.rs | 51 +-- .../src/credentials/main/insert_principal.rs | 49 +++ .../src/credentials/main/mod.rs | 2 + .../src/credentials/models.rs | 80 ++++ .../main/change_person_trigger.rs | 58 +++ .../src/declarations/main/mod.rs | 1 + crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/runs/_helpers/lifecycle/mod.rs | 1 + .../src/runs/_helpers/lifecycle/reads.rs | 25 ++ .../src/runs/_helpers/requests/manual_job.rs | 94 +++++ .../src/runs/_helpers/requests/mod.rs | 1 + .../src/runs/main/list_person_runs.rs | 32 ++ .../src/runs/main/list_runs.rs | 14 +- .../src/runs/main/mod.rs | 2 + .../src/runs/main/request_job_run.rs | 55 +-- .../src/runs/main/request_person_job_run.rs | 23 ++ .../sqlbuild-orchestration/src/runs/models.rs | 2 +- .../src/store/_helpers/migration.rs | 2 + .../store/administrator_initialization.sql | 19 + .../src/store/errors.rs | 5 + .../src/store/main/migrate_store.rs | 9 +- .../tests/api/test_location_work.rs | 3 +- .../tests/api/warehouse_metadata/helpers.rs | 4 +- .../tests/credentials/test_administrators.rs | 390 ++++++++++++++++++ .../tests/credentials/test_contracts.rs | 2 + .../tests/credentials/test_types.rs | 6 + .../tests/runs/test_contracts.rs | 2 + .../tests/runs/test_permissions.rs | 261 ++++++++++++ .../tests/runs/test_types.rs | 8 + .../tests/sensors/test_execution_bounds.rs | 42 +- .../tests/sensors/test_runs.rs | 2 + .../tests/snapshots/helpers.rs | 3 +- deploy/README.md | 29 +- .../compiled_project/publication/helpers.py | 21 +- .../publication/person/helpers.py | 84 ++++ .../publication/test_person_journey.py | 205 +++++++++ 64 files changed, 2661 insertions(+), 160 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs create mode 100644 crates/sqlbuild-orchestration/src/store/administrator_initialization.sql create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_permissions.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json new file mode 100644 index 0000000000..212ca0c21c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -0,0 +1,17 @@ +{ + "first_person": "orders-admin", + "second_person": "customers-admin", + "password": "orders-bootstrap-password-2026", + "disabled_person_sequence": ["assign", "disable", "revoke", "enable", "deny_run"], + "job_settings": [ + {"full_refresh": false}, + {"full_refresh": true}, + {"start_cursor_int": "3", "end_cursor_int": "8"}, + {"start_cursor": "3"}, + {"end_cursor": "8"}, + {"start_cursor_ts": "2026-01-03T00:00:00Z"}, + {"end_cursor_ts": "2026-01-08T00:00:00Z"}, + {"start_cursor_int": "3"}, + {"end_cursor_int": "8"} + ] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 6036987cce..42d8082962 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1,5 +1,37 @@ // Generated from sqb server --openapi. Run npm run generate; do not edit. export interface paths { + "/api/admin/people/{principal_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["changeAdministrator"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/admin/roles": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["assignBuiltinRole"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/auth/logout": { parameters: { query?: never; @@ -115,6 +147,54 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["listPersonRuns"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestJobRun"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/triggers": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["changeTrigger"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/health": { parameters: { query?: never; @@ -439,6 +519,11 @@ export interface paths { export type webhooks = Record; export interface components { schemas: { + /** + * @description Administrator membership and person eligibility share the final-admin invariant. + * @enum {string} + */ + AdministratorChange: "grant" | "revoke" | "disable" | "enable"; ApiErrorBody: { error: string; }; @@ -447,6 +532,8 @@ export interface components { decision_id: string; error: string; }; + /** @enum {string} */ + BuiltinRole: "viewer" | "operator" | "developer" | "admin"; CatchUp: "latest" | { /** @description Process the oldest bounded batch; later due times remain due for the next transaction. */ all: { @@ -520,6 +607,19 @@ export interface components { failed: components["schemas"]["Table"][]; failure?: string | null; }; + /** @description A trusted operator names a published job; execution inputs are never supplied by the caller. */ + JobRunRequest: { + job_key: string; + /** Format: uuid */ + location_id: string; + /** Format: int64 */ + priority: number; + reason: string; + /** Format: uuid */ + run_id: string; + target: string; + window?: null | components["schemas"]["RunWindow"]; + }; /** @description Locations report bounded facts, never connection strings or user exception text. */ LocationHealth: { /** @enum {string} */ @@ -720,6 +820,15 @@ export interface components { kind: "blob"; value: number[]; }; + RoleAssignmentRequest: { + /** Format: uuid */ + location_id?: string | null; + /** Format: uuid */ + principal_id: string; + revoke?: boolean; + role: components["schemas"]["BuiltinRole"]; + target?: string | null; + }; /** @description This is a server-owned instruction, not an arbitrary status supplied by a worker. */ RunChange: { container_id?: string | null; @@ -1132,6 +1241,90 @@ export interface components { } export type $defs = Record; export interface operations { + changeAdministrator: { + parameters: { + query?: never; + header: { + /** @description Session CSRF token */ + "x-sqb-csrf": string; + }; + path: { + principal_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AdministratorChange"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + assignBuiltinRole: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RoleAssignmentRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; logoutSession: { parameters: { query?: never; @@ -1345,6 +1538,125 @@ export interface operations { }; }; }; + listPersonRuns: { + parameters: { + query?: { + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunRecord"][]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + requestJobRun: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["JobRunRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + changeTrigger: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TriggerChangeRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; read_location_health: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index ad96be4dc1..b5fd08acbf 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -340,6 +340,18 @@ ] } }, + { + "name": "create-user", + "parser": { + "prog": "sqb admin create-user", + "description": "Initialize the first installation administrator through local control-store ownership. Available once only; losing usable administrators never reopens bootstrap. Password provisioning is separate.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "NAME", "required": true, "help": "first administrator's person name", "group": 0} + ] + } + }, { "name": "issue-location-token", "parser": { @@ -923,6 +935,11 @@ "metavar": "set-password", "help": "set an active person's password without granting permissions" }, + { + "dest": "create-user", + "metavar": "create-user", + "help": "initialize the first installation administrator once through local store ownership" + }, { "dest": "issue-location-token", "metavar": "issue-location-token", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index e27f177e38..0681d30b12 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -51,6 +51,12 @@ pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { pub(crate) fn control_admin(namespace: &Namespace) -> Result { let value = text(namespace, "admin_subject").unwrap_or_default(); let action = text(namespace, "admin_command"); + if action.as_deref() == Some("create-user") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("user name is required".into())); + } + return Ok(ControlAdmin::BootstrapUser(value)); + } if action.as_deref() == Some("create-location") { if value.trim().is_empty() { return Err(ControlError::Invalid("location name is required".into())); diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 468ad81089..8487fef74a 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -1,6 +1,7 @@ //! Direct database ownership authorizes bootstrap, not an untrusted HTTP request or supplied role. use serde_json::{Value, json}; +use sqlbuild_orchestration::authorization::main::bootstrap_user::bootstrap_user; use sqlbuild_orchestration::credentials::main::create_location::create_location; use sqlbuild_orchestration::credentials::main::create_principal::create_principal; use sqlbuild_orchestration::credentials::main::issue_token::issue_token; @@ -35,6 +36,10 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let id = bootstrap_user(&store, &context, &name).await?; + Ok(json!({"principal_id":id,"name":name,"kind":"person","role":"admin"})) + } ControlAdmin::CreateLocation(name) => { let id = create_location(&store, &name, &context).await?; Ok(json!({"location_id": id, "name": name})) diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index d940e6fe67..44eb36eec1 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -17,6 +17,7 @@ use crate::native_commands::types::LocationPublication; pub(crate) enum ControlAdmin { CreateLocation(String), CreatePrincipal(String, PrincipalKind), + BootstrapUser(String), IssueToken(CredentialScope, DateTime), RevokeToken(Uuid), RotateToken(Uuid), diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs index 37e9631d84..d48b3e763b 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs @@ -5,9 +5,79 @@ use std::time::Duration; use reqwest::{StatusCode, header}; use serde_json::{Value, json}; -use crate::admin_helpers::{administer, invoke_password}; +use crate::admin_helpers::{administer, invoke_admin, invoke_password}; use crate::helpers::{api_ready, start_control, wait_for_exit}; -use crate::test_types::{PasswordCliTestCase, PasswordOriginTestCase}; +use crate::test_types::{AdministratorCliTestCase, PasswordCliTestCase, PasswordOriginTestCase}; + +#[test] +fn given_local_store_ownership_when_create_user_runs_then_only_the_first_process_initializes_an_admin() + { + let test_cases = [AdministratorCliTestCase { + description: "once-only owner bootstrap and authenticated administrator changes", + expected_administrators: 1, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server","--port","0","--no-scheduler","--password-origin","https://control.example.com"]); + let url = api_ready(&mut control).await; + let ordinary = administer(&control,&["create-principal","orders-reader"]); + let first = administer(&control,&["create-user","orders-admin"]); + assert_eq!(first["role"],"admin"); + let connection = rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")).expect("inspect initialization"); + let (count,id): (i64,String) = connection.query_row("SELECT COUNT(*),MIN(principal_id) FROM sqb_role_assignments WHERE role_key='admin' AND revoked_at IS NULL",[],|row| Ok((row.get(0)?,row.get(1)?))).expect("initial administrator membership"); + assert_eq!(count,test_case.expected_administrators,"{}",test_case.description); + assert_eq!(id,first["principal_id"].as_str().expect("initial administrator UUID")); + assert_ne!(id,ordinary["principal_id"].as_str().expect("ordinary person UUID")); + let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().expect("HTTP client"); + let mut sessions = Vec::new(); + for person in [&first,&ordinary] { + let output = invoke_password(&control,person["principal_id"].as_str().expect("person UUID"),b"orders-admin-password-2026"); + assert!(output.status.success()); + let response = client.post(format!("{url}/api/auth/password/login")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .json(&json!({"username":person["name"],"password":"orders-admin-password-2026"})).send().await.expect("password login"); + assert_eq!(response.status(),StatusCode::OK); + let cookie = response.headers().get_all(header::SET_COOKIE).iter().filter_map(|value| value.to_str().ok()) + .find(|value| value.starts_with("__Host-sqb-session=")).expect("session cookie").split(';').next().expect("cookie pair").to_owned(); + let body: Value = response.json().await.expect("session body"); + sessions.push((cookie,body["csrf_token"].as_str().expect("CSRF token").to_owned())); + } + let ordinary_id = ordinary["principal_id"].as_str().expect("ordinary person UUID"); + for (actor,target,action,expected) in [ + (1,ordinary_id,"grant",StatusCode::FORBIDDEN), + (0,id.as_str(),"revoke",StatusCode::FORBIDDEN), + (0,ordinary_id,"grant",StatusCode::OK), + (1,id.as_str(),"revoke",StatusCode::OK), + (0,id.as_str(),"grant",StatusCode::FORBIDDEN), + (1,ordinary_id,"disable",StatusCode::FORBIDDEN), + ] { + let response = client.post(format!("{url}/api/admin/people/{target}")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .header(header::COOKIE,&sessions[actor].0).header("x-sqb-csrf",&sessions[actor].1) + .json(&action).send().await.expect("administrator mutation response"); + assert_eq!(response.status(),expected,"{action} by actor {actor}"); + let body: Value = response.json().await.expect("mutation result"); + assert_eq!(body["applied"].as_bool(), (expected == StatusCode::OK).then_some(true)); + assert_eq!(body["error"].as_str().is_some_and(|message| message.contains("decision")),expected == StatusCode::FORBIDDEN); + } + for disabled_at in [None,Some("2026-01-01T00:00:00Z")] { + connection.execute("UPDATE sqb_principals SET disabled_at=?1",[disabled_at]).expect("simulate out-of-band administrator loss"); + let rejected = invoke_admin(&control,&["create-user","replacement-admin"]); + assert!(!rejected.status.success()); + assert!(rejected.stdout.is_empty()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("bootstrap is closed")); + } + let initialized: i64 = connection.query_row("SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.initialized' AND actor='local-control-store-owner'",[],|row| row.get(0)).expect("bootstrap audit"); + assert_eq!(initialized,1); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} #[test] fn given_owner_provisioned_passwords_when_a_standalone_server_authenticates_then_resets_invalidate_sessions_without_granting_authority() diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 517e7a2d83..94d4af3641 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -41,6 +41,11 @@ pub(crate) struct PasswordOriginTestCase { pub(crate) expected_error: &'static str, } +pub(crate) struct AdministratorCliTestCase { + pub(crate) description: &'static str, + pub(crate) expected_administrators: i64, +} + pub(crate) struct ProxyCliTestCase { pub(crate) description: &'static str, pub(crate) subject: &'static str, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 15b60a9a2e..5256220d30 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -37,6 +37,26 @@ pub(crate) fn router( .route("/api/auth/session", get(current)) .route("/api/auth/logout", post(logout)) .route("/api/auth/password", post(change_password)) + .route( + "/api/admin/people/{principal_id}", + post(crate::api::_helpers::resources::administrators::change), + ) + .route( + "/api/admin/roles", + post(crate::api::_helpers::resources::administrators::assign), + ) + .route( + "/api/control/runs", + post(crate::api::_helpers::resources::person_operations::request_job), + ) + .route( + "/api/control/triggers", + post(crate::api::_helpers::resources::person_operations::change_trigger), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/runs", + get(crate::api::_helpers::resources::person_operations::recent_runs), + ) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); Ok(protected .route("/api/auth/password/login", post(login)) diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs index 61d02c1491..68256046ad 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -130,6 +130,13 @@ pub(crate) fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&' values.next().is_none().then_some(value) } +pub(crate) fn session_credentials(headers: &HeaderMap) -> (&str, &str) { + ( + browser_cookie(headers, PASSWORD_COOKIE).unwrap_or_default(), + single_header(headers, PASSWORD_CSRF_HEADER).unwrap_or_default(), + ) +} + pub(crate) fn browser_cookie<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { let mut session = None; for header in headers.get_all(header::COOKIE) { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs new file mode 100644 index 0000000000..320f74ca76 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -0,0 +1,51 @@ +//! Reauthenticate membership changes inside the serialized mutation, not only in middleware. + +use axum::extract::{Path, State}; +use axum::http::HeaderMap; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::password_transport::session_credentials; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::authorization::main::assign_builtin_role::assign_builtin_role; +use crate::authorization::main::change_administrator::change_administrator; +use crate::credentials::models::{AdministratorChange, AuditContext, RoleAssignmentRequest}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/admin/people/{principal_id}", operation_id = "changeAdministrator", + params(("principal_id" = Uuid, Path), ("x-sqb-csrf" = String, Header, description = "Session CSRF token")), + request_body = AdministratorChange, + responses((status = 200, body = MutationResponse), (status = 401, body = crate::api::models::AuthenticationDenial), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn change( + (State(store), Extension(context)): (State, Extension), + Path(principal_id): Path, + headers: HeaderMap, + Json(change): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: change_administrator( + &store, + &context, + session_credentials(&headers), + (principal_id, change), + ) + .await?, + })) +} + +#[utoipa::path(post, path = "/api/admin/roles", operation_id = "assignBuiltinRole", + params(("x-sqb-csrf" = String, Header)), request_body = RoleAssignmentRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn assign( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: assign_builtin_role(&store, (&context, session_credentials(&headers)), &request) + .await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index d241beee10..80a79e5e21 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -1,8 +1,10 @@ +pub(crate) mod administrators; pub(crate) mod changes; pub(crate) mod client; pub(crate) mod docker_image; pub(crate) mod location_health; pub(crate) mod location_work; +pub(crate) mod person_operations; pub(crate) mod router; pub(crate) mod run_reads; pub(crate) mod runs; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs new file mode 100644 index 0000000000..bc6296fef1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -0,0 +1,70 @@ +//! Person operations use browser authentication, never the machine-resource bearer router. + +use axum::extract::{Path, Query, State}; +use axum::http::HeaderMap; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::password_transport::session_credentials; +use crate::api::errors::ApiError; +use crate::api::models::{MutationResponse, ReadQuery}; +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_person_trigger::change_person_trigger; +use crate::declarations::models::TriggerChangeRequest; +use crate::runs::main::list_person_runs::list_person_runs; +use crate::runs::main::request_person_job_run::request_person_job_run; +use crate::runs::models::{JobRunRequest, RunRecord}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", + params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn request_job( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_person_job_run( + &store, + (&context, session_credentials(&headers)), + &request, + ) + .await?, + })) +} + +#[utoipa::path(post, path = "/api/control/triggers", operation_id = "changeTrigger", + params(("x-sqb-csrf" = String, Header)), request_body = TriggerChangeRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn change_trigger( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: change_person_trigger(&store, (&context, session_credentials(&headers)), &request) + .await?, + })) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs", operation_id = "listPersonRuns", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("limit" = Option, Query)), + responses((status = 200, body = Vec), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn recent_runs( + (State(store), Extension(context)): (State, Extension), + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + headers: HeaderMap, +) -> Result>, ApiError> { + Ok(Json( + list_person_runs( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, options.limit.unwrap_or(100)), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs index ea9da3591d..f5e61eda14 100644 --- a/crates/sqlbuild-orchestration/src/api/errors.rs +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -30,6 +30,13 @@ impl From for ApiError { status: StatusCode::BAD_REQUEST, message, }, + ControlError::Denied { + reason, + decision_id, + } => Self { + status: StatusCode::FORBIDDEN, + message: format!("{reason}; decision {decision_id}"), + }, ControlError::StaleGeneration | ControlError::Fenced => Self { status: StatusCode::CONFLICT, message: error.to_string(), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 621d56390f..4d6984d45f 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -6,12 +6,17 @@ use crate::api::_helpers::authentication::oidc; use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::trusted_proxy; use crate::api::_helpers::resources::{ - changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, - warehouse_metadata, + administrators, changes, location_health, location_work, person_operations, run_reads, runs, + snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( + administrators::change, + administrators::assign, + person_operations::request_job, + person_operations::change_trigger, + person_operations::recent_runs, password_sessions::login, password_sessions::current, password_sessions::logout, diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs new file mode 100644 index 0000000000..799b864352 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs @@ -0,0 +1,105 @@ +//! Apply eligibility/membership changes without releasing the caller's authorization lock. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrators::{ + deny_change, grant_installation_admin, remaining_administrator, +}; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AdministratorChange, AuditContext}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn apply_change( + transaction: ControlTransaction, + context: &AuditContext, + (principal_id, change): (Uuid, AdministratorChange), +) -> Result { + let (mut transaction, rows) = query(transaction, + "SELECT p.disabled_at,a.assignment_id FROM sqb_principals p LEFT JOIN sqb_role_assignments a \ + ON p.principal_id=a.principal_id AND a.role_key='admin' AND a.location_id IS NULL \ + AND a.target IS NULL AND a.revoked_at IS NULL WHERE p.principal_id=$1 AND p.kind='person'", + vec![Parameter::Id(principal_id)]).await?; + let Some(person) = rows.first() else { + return deny_change( + transaction, + context, + principal_id, + "target must be a person", + ) + .await; + }; + let active = person["disabled_at"].is_null(); + let administrator = !person["assignment_id"].is_null(); + if active + && administrator + && matches!( + change, + AdministratorChange::Revoke | AdministratorChange::Disable + ) + { + let other; + (transaction, other) = remaining_administrator(transaction, principal_id).await?; + if !other { + return deny_change( + transaction, + context, + principal_id, + "cannot remove or disable the final usable installation administrator", + ) + .await; + } + } + let now = transaction.now; + + let changed; + let action = match change { + AdministratorChange::Grant => { + if !active { + return deny_change( + transaction, + context, + principal_id, + "administrator must be an active person", + ) + .await; + } + let applied; + (transaction, applied) = + grant_installation_admin(transaction, context, principal_id, "manual").await?; + commit(transaction).await?; + return Ok(applied); + } + AdministratorChange::Revoke => { + (transaction, changed) = query(transaction, + "UPDATE sqb_role_assignments SET revoked_at=$1,revoked_by=$2,active_key=NULL \ + WHERE principal_id=$3 AND role_key='admin' AND location_id IS NULL AND target IS NULL \ + AND revoked_at IS NULL RETURNING assignment_id", + vec![Parameter::Time(now), Parameter::Text(context.actor.clone()),Parameter::Id(principal_id)]).await?; + "role.revoked" + } + AdministratorChange::Disable | AdministratorChange::Enable => { + let disable = change == AdministratorChange::Disable; + let predicate = if disable { "IS NULL" } else { "IS NOT NULL" }; + (transaction, changed) = query(transaction, + &format!("UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2 AND disabled_at {predicate} RETURNING principal_id"), + vec![Parameter::OptionalTime(disable.then_some(now)),Parameter::Id(principal_id)]).await?; + if disable { + "principal.disabled" + } else { + "principal.enabled" + } + } + }; + let applied = !changed.is_empty(); + if applied { + transaction = append_audit(transaction, context, (Uuid::now_v7(), action), + (json!({"principal_id":principal_id}), json!({"administrator":administrator,"enabled":active}), + json!({"administrator":administrator && change!=AdministratorChange::Revoke,"enabled":change==AdministratorChange::Enable || (active && change!=AdministratorChange::Disable)}))).await?; + transaction.changes.insert(Change::Credentials); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs new file mode 100644 index 0000000000..4e4e395a2b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs @@ -0,0 +1,115 @@ +//! Authenticated installation administrators assign additive built-ins; code never assigns people. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrator_changes::apply_change; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{ + AdministratorChange, AuditContext, BuiltinRole, RoleAssignmentRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn assign_role_in( + transaction: ControlTransaction, + context: &AuditContext, + request: &RoleAssignmentRequest, +) -> Result { + if request + .target + .as_ref() + .is_some_and(|target| target.trim().is_empty()) + || (request.target.is_some() && request.location_id.is_none()) + { + return Err(ControlError::Invalid( + "role target requires a location and a nonempty name".into(), + )); + } + if request.role == BuiltinRole::Admin { + if request.location_id.is_some() || request.target.is_some() { + return deny( + transaction, + context, + json!(request), + "administrator assignment must be installation-wide", + ) + .await; + } + return apply_change( + transaction, + context, + ( + request.principal_id, + if request.revoke { + AdministratorChange::Revoke + } else { + AdministratorChange::Grant + }, + ), + ) + .await; + } + let (transaction, people) = query(transaction, "SELECT principal_id,disabled_at FROM sqb_principals WHERE principal_id=$1 AND kind='person'", + vec![Parameter::Id(request.principal_id)]).await?; + if people + .first() + .is_none_or(|person| !request.revoke && !person["disabled_at"].is_null()) + { + return deny( + transaction, + context, + json!(request), + "role assignment requires an enabled person; revocation requires an existing person", + ) + .await; + } + assign_scoped_role(transaction, context, request).await +} + +async fn assign_scoped_role( + transaction: ControlTransaction, + context: &AuditContext, + request: &RoleAssignmentRequest, +) -> Result { + let role = serde_json::to_value(request.role) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let key = serde_json::to_string(&( + request.principal_id, + request.role, + request.location_id, + &request.target, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let assignment_id = Uuid::now_v7(); + let now = transaction.now; + let (mut transaction, changed) = if request.revoke { + query(transaction, "UPDATE sqb_role_assignments SET revoked_at=$1,revoked_by=$2,active_key=NULL WHERE active_key=$3 RETURNING assignment_id", + vec![Parameter::Time(now),Parameter::Text(context.actor.clone()),Parameter::Text(key)]).await? + } else { + query(transaction, "INSERT INTO sqb_role_assignments (assignment_id,principal_id,role_key,location_id,target,source,granted_by,granted_at,active_key) VALUES ($1,$2,$3,$4,$5,'manual',$6,$7,$8) ON CONFLICT (active_key) DO NOTHING RETURNING assignment_id", + vec![Parameter::Id(assignment_id),Parameter::Id(request.principal_id),Parameter::Text(role.as_str().unwrap_or_default().into()),Parameter::OptionalId(request.location_id),Parameter::OptionalText(request.target.clone()),Parameter::Text(context.actor.clone()),Parameter::Time(now),Parameter::Text(key)]).await? + }; + let applied = !changed.is_empty(); + if applied { + transaction = append_audit( + transaction, + context, + ( + assignment_id, + if request.revoke { + "role.revoked" + } else { + "role.assigned" + }, + ), + (json!(request), json!(null), json!({"assignments":changed})), + ) + .await?; + transaction.changes.insert(Change::Credentials); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs new file mode 100644 index 0000000000..95db59c244 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs @@ -0,0 +1,79 @@ +//! Installation administrator membership; callers establish bootstrap or authenticated authority. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn grant_installation_admin( + transaction: ControlTransaction, + context: &AuditContext, + principal_id: Uuid, + source: &str, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, people) = query(transaction, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND kind='person' AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)]).await?; + if people.is_empty() { + return Err(ControlError::Invalid( + "administrator must be an active person".into(), + )); + } + let assignment_id = Uuid::now_v7(); + let now = transaction.now; + let (mut transaction, rows) = query(transaction, + "INSERT INTO sqb_role_assignments (assignment_id,principal_id,role_key,source,granted_by,granted_at,active_key) \ + VALUES ($1,$2,'admin',$3,$4,$5,$6) ON CONFLICT (active_key) DO NOTHING RETURNING assignment_id", + vec![Parameter::Id(assignment_id), Parameter::Id(principal_id), Parameter::Text(source.into()), + Parameter::Text(context.actor.clone()), Parameter::Time(now), + Parameter::Text(format!("{principal_id}:admin:installation"))]).await?; + let applied = !rows.is_empty(); + if applied { + transaction = append_audit( + transaction, + context, + (assignment_id, "role.assigned"), + ( + json!({"principal_id": principal_id}), + json!(null), + json!({"role_key":"admin", "source":source, "assignment_id":assignment_id}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + } + Ok((transaction, applied)) +} + +pub(crate) async fn remaining_administrator( + transaction: ControlTransaction, + principal_id: Uuid, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT a.principal_id FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) \ + JOIN sqb_password_credentials c ON c.principal_id=p.principal_id \ + WHERE a.role_key='admin' AND a.location_id IS NULL AND a.target IS NULL AND a.revoked_at IS NULL \ + AND p.kind='person' AND p.disabled_at IS NULL AND p.principal_id<>$1 LIMIT 1", + vec![Parameter::Id(principal_id)]).await?; + Ok((transaction, !rows.is_empty())) +} + +pub(crate) async fn deny_change( + transaction: ControlTransaction, + context: &AuditContext, + principal_id: Uuid, + reason: &str, +) -> Result { + deny( + transaction, + context, + json!({"principal_id":principal_id}), + reason, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs new file mode 100644 index 0000000000..b5107f4f02 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod administrator_changes; +pub(crate) mod administrator_roles; +pub(crate) mod administrators; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs new file mode 100644 index 0000000000..43622bf833 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs @@ -0,0 +1,27 @@ +//! Only live installation administrators may assign or revoke additive built-in roles. + +use crate::authorization::_helpers::administrator_roles::assign_role_in; +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, PermissionScope, RoleAssignmentRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn assign_builtin_role( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + request: &RoleAssignmentRequest, +) -> Result { + let (transaction, context) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::RoleAssign, + location_id: None, + target: None, + nodes: Default::default(), + }, + ) + .await?; + assign_role_in(transaction, &context, request).await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs new file mode 100644 index 0000000000..8ee18a3cff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs @@ -0,0 +1,93 @@ +//! Live identity and additive built-in grants are evaluated under the operation's store lock. + +use serde_json::json; + +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::main::authenticate_session_in::authenticate_session_in; +use crate::credentials::models::{ + Action, AuditContext, BuiltinRole, PermissionScope, SessionAccess, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +fn role_allows(role: BuiltinRole, action: Action) -> bool { + use Action::{ + ConnectionTest, JobRun, PlanPreview, RunAdhoc, RunCancel, RunRerun, TriggerToggle, View, + ViewSql, + }; + match role { + BuiltinRole::Admin => true, + BuiltinRole::Viewer => action == View, + BuiltinRole::Operator => { + matches!(action, View | JobRun | RunCancel | RunRerun | TriggerToggle) + } + BuiltinRole::Developer => matches!( + action, + View | JobRun + | RunCancel + | RunRerun + | TriggerToggle + | ViewSql + | RunAdhoc + | ConnectionTest + | PlanPreview + ), + } +} + +pub(crate) async fn authorize_session( + transaction: ControlTransaction, + (context, credentials): (&AuditContext, (&str, &str)), + scope: &PermissionScope, +) -> Result<(ControlTransaction, AuditContext), ControlError> { + let access = match scope.action { + Action::View | Action::ViewSql | Action::ViewEvidence | Action::AuditView => { + SessionAccess::Read + } + _ => SessionAccess::Write { + csrf: credentials.1, + }, + }; + let (transaction, identity) = + authenticate_session_in(transaction, credentials.0, access).await?; + let mut context = AuditContext { + actor: identity.as_ref().map_or_else( + || "unauthenticated".into(), + |identity| format!("principal:{}", identity.principal_id), + ), + granting_role: "none".into(), + ..context.clone() + }; + let Some(identity) = identity else { + return deny( + transaction, + &context, + json!(scope), + "authenticated person session and CSRF required", + ) + .await; + }; + let (transaction, rows) = query(transaction, + "SELECT role_key,location_id,target FROM sqb_role_assignments WHERE principal_id=$1 AND revoked_at IS NULL ORDER BY role_key", + vec![Parameter::Id(identity.principal_id)]).await?; + for row in rows { + let Ok(role) = serde_json::from_value::(row["role_key"].clone()) else { + continue; + }; + if role_allows(role, scope.action) + && (row["location_id"].is_null() || row["location_id"] == json!(scope.location_id)) + && (row["target"].is_null() || row["target"] == json!(scope.target)) + { + context.granting_role = row["role_key"].as_str().unwrap_or_default().into(); + return Ok((transaction, context)); + } + } + deny( + transaction, + &context, + json!(scope), + "missing permission in this scope; all requested nodes denied", + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs b/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs new file mode 100644 index 0000000000..2e12d36608 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs @@ -0,0 +1,56 @@ +//! Direct local store ownership may initialize the first administrator once, never reopen recovery. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrators::grant_installation_admin; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::insert_principal::insert_principal; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn bootstrap_user( + store: &ControlStore, + context: &AuditContext, + name: &str, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Write).await?, + "SELECT principal_id FROM sqb_admin_initialization WHERE singleton=1", + vec![], + ) + .await?; + if !rows.is_empty() { + return Err(ControlError::Invalid( + "administrator initialization already occurred; bootstrap is closed".into(), + )); + } + let (transaction, principal_id) = + insert_principal(transaction, context, (name, PrincipalKind::Person)).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_admin_initialization (singleton,principal_id,initialized_at) VALUES (1,$1,$2)", + vec![Parameter::Id(principal_id), Parameter::Time(now)]).await?; + let (transaction, _) = + grant_installation_admin(transaction, context, principal_id, "bootstrap").await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&principal_id, b"administrator-initialized"), + "admin.initialized", + ), + ( + json!({"installation":true}), + json!(null), + json!({"principal_id":principal_id}), + ), + ) + .await?, + ) + .await?; + Ok(principal_id) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs new file mode 100644 index 0000000000..ffd44f1494 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs @@ -0,0 +1,33 @@ +//! Live administrator/session checks, final-admin protection, audit and revision share one lock. + +use uuid::Uuid; + +use crate::authorization::_helpers::administrator_changes::apply_change; +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AdministratorChange, AuditContext, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn change_administrator( + store: &ControlStore, + context: &AuditContext, + (bearer, csrf): (&str, &str), + (principal_id, change): (Uuid, AdministratorChange), +) -> Result { + let (transaction, context) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, (bearer, csrf)), + &PermissionScope { + action: match change { + AdministratorChange::Grant | AdministratorChange::Revoke => Action::RoleAssign, + AdministratorChange::Disable | AdministratorChange::Enable => Action::UserManage, + }, + location_id: None, + target: None, + nodes: Default::default(), + }, + ) + .await?; + apply_change(transaction, &context, (principal_id, change)).await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs b/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs new file mode 100644 index 0000000000..403164f51f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs @@ -0,0 +1,37 @@ +//! A denial is returned only after its decision ID and authoritative scope commit durably. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlTransaction; + +pub(crate) async fn deny( + transaction: ControlTransaction, + context: &AuditContext, + scope: Value, + reason: &str, +) -> Result { + let decision_id = Uuid::now_v7(); + commit( + append_audit( + transaction, + context, + (decision_id, "authorization.denied"), + ( + scope, + json!(null), + json!({"decision_id":decision_id,"allowed":false,"reason":reason}), + ), + ) + .await?, + ) + .await?; + Err(ControlError::Denied { + reason: reason.into(), + decision_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs new file mode 100644 index 0000000000..ec32d511b9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -0,0 +1,5 @@ +pub mod assign_builtin_role; +pub(crate) mod authorize_session; +pub mod bootstrap_user; +pub mod change_administrator; +pub(crate) mod deny_authorization; diff --git a/crates/sqlbuild-orchestration/src/authorization/mod.rs b/crates/sqlbuild-orchestration/src/authorization/mod.rs new file mode 100644 index 0000000000..d1ddfc554a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/mod.rs @@ -0,0 +1,4 @@ +//! Closed, scoped person permissions and serialized installation-administrator lifecycle. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs index d9fa231835..ee30024156 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs @@ -1,36 +1,22 @@ -//! Live password versions invalidate all earlier sessions without per-process caches. +//! Read authentication and mutations share the same live session/version checks. -use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; -use crate::credentials::models::{SessionAccess, SessionIdentity, StoredSession}; +use crate::credentials::_helpers::secrets::bearer_id; +use crate::credentials::main::authenticate_session_in::authenticate_session_in as authenticate; +use crate::credentials::models::{SessionAccess, SessionIdentity}; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn authenticate_session( store: &ControlStore, bearer: &str, access: SessionAccess<'_>, ) -> Result, ControlError> { - let Some(id) = bearer_id(bearer) else { + if bearer_id(bearer).is_none() { return Ok(None); - }; - let transaction = begin(store, TransactionMode::Read).await?; - let now = transaction.now; - let (transaction, rows) = query(transaction, - "SELECT s.session_id, s.principal_id, s.expires_at, s.secret_hash, s.csrf_hash \ - FROM sqb_password_sessions s JOIN sqb_principals p ON p.principal_id=s.principal_id \ - JOIN sqb_password_credentials c ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ - WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ - AND p.kind='person' AND p.disabled_at IS NULL", - vec![Parameter::Id(id), Parameter::Time(now)]).await?; + } + let (transaction, identity) = + authenticate(begin(store, TransactionMode::Read).await?, bearer, access).await?; commit(transaction).await?; - let Some(row) = rows.into_iter().next() else { - return Ok(None); - }; - let session: StoredSession = decode_row(row)?; - let csrf_matches = match access { - SessionAccess::Read => true, - SessionAccess::Write { csrf } => secret_matches(csrf, &session.csrf_hash), - }; - Ok((secret_matches(bearer, &session.secret_hash) && csrf_matches).then_some(session.identity)) + Ok(identity) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs new file mode 100644 index 0000000000..67b8caa435 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs @@ -0,0 +1,37 @@ +//! Live password versions invalidate all earlier sessions without per-process caches. + +use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; +use crate::credentials::models::{SessionAccess, SessionIdentity, StoredSession}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn authenticate_session_in( + transaction: ControlTransaction, + bearer: &str, + access: SessionAccess<'_>, +) -> Result<(ControlTransaction, Option), ControlError> { + let Some(id) = bearer_id(bearer) else { + return Ok((transaction, None)); + }; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.session_id, s.principal_id, s.expires_at, s.secret_hash, s.csrf_hash \ + FROM sqb_password_sessions s JOIN sqb_principals p ON p.principal_id=s.principal_id \ + JOIN sqb_password_credentials c ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ + WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ + AND p.kind='person' AND p.disabled_at IS NULL", + vec![Parameter::Id(id), Parameter::Time(now)]).await?; + let Some(row) = rows.into_iter().next() else { + return Ok((transaction, None)); + }; + let session: StoredSession = decode_row(row)?; + let csrf_matches = match access { + SessionAccess::Read => true, + SessionAccess::Write { csrf } => secret_matches(csrf, &session.csrf_hash), + }; + Ok(( + transaction, + (secret_matches(bearer, &session.secret_hash) && csrf_matches).then_some(session.identity), + )) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs index 4b85936350..153fd94d6e 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs @@ -1,55 +1,24 @@ -//! Server-owned people and service accounts; creating one never grants a role implicitly. +//! Creating a person or service account never implicitly assigns a role. -use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::insert_principal::insert_principal; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn create_principal( store: &ControlStore, context: &AuditContext, - (name, kind): (&str, PrincipalKind), + principal: (&str, PrincipalKind), ) -> Result { - if name.trim().is_empty() { - return Err(ControlError::Invalid("principal name is required".into())); - } - let transaction = begin(store, TransactionMode::Write).await?; - let id = Uuid::now_v7(); - let now = transaction.now; - let kind = match kind { - PrincipalKind::Person => "person", - PrincipalKind::ServiceAccount => "service_account", - }; - let (transaction, _) = query( - transaction, - "INSERT INTO sqb_principals (principal_id, name, kind, created_at) VALUES ($1,$2,$3,$4)", - vec![ - Parameter::Id(id), - Parameter::Text(name.into()), - Parameter::Text(kind.into()), - Parameter::Time(now), - ], - ) - .await?; - commit( - append_audit( - transaction, - context, - (Uuid::new_v5(&id, b"created"), "principal.created"), - ( - json!({"principal_id": id}), - json!(null), - json!({"name": name, "kind": kind}), - ), - ) - .await?, + let (transaction, id) = insert_principal( + begin(store, TransactionMode::Write).await?, + context, + principal, ) .await?; + commit(transaction).await?; Ok(id) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs new file mode 100644 index 0000000000..d8d2c95e5b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs @@ -0,0 +1,49 @@ +//! Create a role-free identity; bootstrap authority is a separate, one-time operation. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn insert_principal( + transaction: ControlTransaction, + context: &AuditContext, + (name, kind): (&str, PrincipalKind), +) -> Result<(ControlTransaction, Uuid), ControlError> { + if name.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let id = Uuid::now_v7(); + let now = transaction.now; + let kind = match kind { + PrincipalKind::Person => "person", + PrincipalKind::ServiceAccount => "service_account", + }; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_principals (principal_id, name, kind, created_at) VALUES ($1,$2,$3,$4)", + vec![ + Parameter::Id(id), + Parameter::Text(name.into()), + Parameter::Text(kind.into()), + Parameter::Time(now), + ], + ) + .await?; + let transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "principal.created"), + ( + json!({"principal_id": id}), + json!(null), + json!({"name": name, "kind": kind}), + ), + ) + .await?; + Ok((transaction, id)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index ed61a1d2ae..7416722370 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,12 +1,14 @@ pub(crate) mod append_audit; pub(crate) mod authenticate_bound_person; pub mod authenticate_session; +pub(crate) mod authenticate_session_in; pub mod authenticate_token; pub(crate) mod claim_oidc_flow; pub mod create_location; pub(crate) mod create_oidc_flow; pub mod create_principal; pub mod development_location; +pub(crate) mod insert_principal; pub mod issue_token; pub mod login_password; pub mod record_authentication_denial; diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index 7c9157a42e..e447c8bfcf 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -38,6 +38,76 @@ pub struct AuditContext { pub ip_address: Option, } +/// Closed permission vocabulary; unknown actions fail deserialization. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub enum Action { + #[serde(rename = "view")] + View, + #[serde(rename = "view.sql")] + ViewSql, + #[serde(rename = "view.evidence")] + ViewEvidence, + #[serde(rename = "job.run")] + JobRun, + #[serde(rename = "run.cancel")] + RunCancel, + #[serde(rename = "run.rerun")] + RunRerun, + #[serde(rename = "run.rewrite_history")] + RunRewriteHistory, + #[serde(rename = "trigger.toggle")] + TriggerToggle, + #[serde(rename = "sensor.reset_cursor")] + SensorResetCursor, + #[serde(rename = "warehouse.sync")] + WarehouseSync, + #[serde(rename = "run.adhoc")] + RunAdhoc, + #[serde(rename = "connection.test")] + ConnectionTest, + #[serde(rename = "plan.preview")] + PlanPreview, + #[serde(rename = "location.manage")] + LocationManage, + #[serde(rename = "user.manage")] + UserManage, + #[serde(rename = "role.assign")] + RoleAssign, + #[serde(rename = "token.manage")] + TokenManage, + #[serde(rename = "audit.view")] + AuditView, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum BuiltinRole { + Viewer, + Operator, + Developer, + Admin, +} + +/// Scope is computed by the server from the operation, never accepted as a claimed write set. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct PermissionScope { + pub(crate) action: Action, + pub(crate) location_id: Option, + pub(crate) target: Option, + pub(crate) nodes: std::collections::BTreeSet, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RoleAssignmentRequest { + pub principal_id: Uuid, + pub role: BuiltinRole, + pub location_id: Option, + pub target: Option, + #[serde(default)] + pub revoke: bool, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct TokenIdentity { pub token_id: Uuid, @@ -61,6 +131,16 @@ pub enum PasswordAuthority { }, } +/// Administrator membership and person eligibility share the final-admin invariant. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum AdministratorChange { + Grant, + Revoke, + Disable, + Enable, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct SessionIdentity { pub session_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs new file mode 100644 index 0000000000..e44ae32a92 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs @@ -0,0 +1,58 @@ +//! Resolve a trigger's real location before authorizing an authenticated runtime override. + +use serde_json::json; + +use crate::authorization::main::authorize_session::authorize_session; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::models::{Action, AuditContext, PermissionScope}; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn change_person_trigger( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + request: &TriggerChangeRequest, +) -> Result { + let (transaction, rows) = query(begin(store, TransactionMode::Write).await?, + "SELECT location_id,target FROM sqb_triggers WHERE trigger_id=$1 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)]).await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("current trigger not found".into()))?; + let scope = PermissionScope { + action: match request.change { + TriggerChange::Enable | TriggerChange::Pause | TriggerChange::ResetDefault => { + Action::TriggerToggle + } + TriggerChange::SetCursor { .. } | TriggerChange::ResetCursor => { + Action::SensorResetCursor + } + }, + location_id: serde_json::from_value(row["location_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + target: serde_json::from_value(row["target"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + nodes: Default::default(), + }; + let (transaction, context) = + authorize_session(transaction, (context, credentials), &scope).await?; + if request.reason.trim().is_empty() { + return deny( + transaction, + &context, + json!({"trigger_id":request.trigger_id}), + "trigger override requires a reason", + ) + .await; + } + let request = TriggerChangeRequest { + actor: context.actor.clone(), + ..request.clone() + }; + let (transaction, applied) = change_trigger_in(transaction, &context, &request).await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index 4e0773fdd4..23cab85fba 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -1,3 +1,4 @@ +pub mod change_person_trigger; pub mod change_trigger; pub(crate) mod frozen_definitions; pub mod group_dependencies; diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 3f1421053e..80ef365a70 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub mod api; +pub mod authorization; pub mod credentials; pub mod declarations; pub mod platform; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 0ef9c931db..6f70d90d15 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -5,4 +5,5 @@ pub(in crate::runs) mod nodes; pub(in crate::runs) mod output; pub(in crate::runs) mod projection; pub(in crate::runs) mod publish_change; +pub(in crate::runs) mod reads; pub(in crate::runs) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs new file mode 100644 index 0000000000..4689f8cbff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs @@ -0,0 +1,25 @@ +//! Bounded metadata projection; it never includes SQL, sampled rows or captured output. + +use uuid::Uuid; + +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn list_runs_in( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + limit: u32, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT run_id, location_id, target, snapshot_id, job_key, status, queued_at, launched_at, started_at, ended_at, selection_json, node_counts_json, last_event_storage_id \ + FROM sqb_runs WHERE location_id=$1 AND target=$2 ORDER BY queued_at DESC, run_id DESC LIMIT $3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + Ok(( + transaction, + rows.into_iter() + .map(decode_row) + .collect::, _>>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs new file mode 100644 index 0000000000..acbff53245 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -0,0 +1,94 @@ +//! Manual job freezing, idempotence, authorization and publication share one transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext, PermissionScope}; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::models::{JobRunRequest, RunKind, RunWindow}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn request_job_run_in( + mut transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option<(&str, &str)>), + request: &JobRunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let mut context = context.clone(); + let mut scope = PermissionScope { + action: Action::JobRun, + location_id: Some(request.location_id), + target: Some(request.target.clone()), + nodes: Default::default(), + }; + if let Some(credentials) = credentials { + (transaction, context) = + authorize_session(transaction, (&context, credentials), &scope).await?; + } + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.target.trim().is_empty() + || request.job_key.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "job run needs an actor, granting role, target, job and reason".into(), + )); + } + if request.window.as_ref().is_some_and(|window| match window { + RunWindow::Timestamp { start, end } => start > end, + RunWindow::Integer { start, end } => start > end, + }) { + return Err(ControlError::Invalid( + "backfill window start must not follow end".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); + let (transaction, retry) = request_retry( + transaction, + &context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { + return Ok((transaction, false)); + } + let (mut transaction, frozen) = frozen_job_request(transaction, &context, request).await?; + if let Some(credentials) = credentials { + let mut executions = vec![frozen.clone()]; + if frozen.kind == RunKind::Group { + (transaction, executions) = frozen_group_children(transaction, &frozen).await?; + } + scope.nodes = executions + .iter() + .flat_map(|run| run.write_set.iter().cloned()) + .collect(); + if request.window.is_some() + || executions.iter().any(|run| { + run.settings + .get("full_refresh") + .and_then(serde_json::Value::as_bool) + == Some(true) + || run.settings.contains_key("start_cursor") + || run.settings.contains_key("end_cursor") + || run.settings.contains_key("start_cursor_ts") + || run.settings.contains_key("end_cursor_ts") + || run.settings.contains_key("start_cursor_int") + || run.settings.contains_key("end_cursor_int") + }) + { + scope.action = Action::RunRewriteHistory; + (transaction, context) = + authorize_session(transaction, (&context, credentials), &scope).await?; + } + } + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit(transaction, &context, (event_id, "job.run_requested"), + (json!({"location_id":request.location_id,"target":request.target,"job_key":request.job_key,"run_id":request.run_id}), + json!(null), json!({"request":request,"frozen":frozen}))).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs index 010bce9e3a..0c44cb5dcd 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -1,3 +1,4 @@ pub(in crate::runs) mod job; +pub(in crate::runs) mod manual_job; pub(in crate::runs) mod publication; pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs new file mode 100644 index 0000000000..b2e33af357 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs @@ -0,0 +1,32 @@ +//! Metadata reads hold live permission checks and results in the same transaction. + +use uuid::Uuid; + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, PermissionScope}; +use crate::runs::_helpers::lifecycle::reads::list_runs_in; +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn list_person_runs( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + (location_id, target, limit): (Uuid, &str, u32), +) -> Result, ControlError> { + let (transaction, _) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + ) + .await?; + let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; + commit(transaction).await?; + Ok(runs) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs index 516302af18..b7bf47534c 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs @@ -2,13 +2,12 @@ use uuid::Uuid; +use crate::runs::_helpers::lifecycle::reads::list_runs_in; use crate::runs::models::RunRecord; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::decode_row::decode_row; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; /// Return recent projected runs; per-run events remain in Recall. pub async fn list_runs( @@ -17,14 +16,7 @@ pub async fn list_runs( limit: u32, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, rows) = query(transaction, - "SELECT run_id, location_id, target, snapshot_id, job_key, status, queued_at, launched_at, started_at, ended_at, selection_json, node_counts_json, last_event_storage_id \ - FROM sqb_runs WHERE location_id=$1 AND target=$2 ORDER BY queued_at DESC, run_id DESC LIMIT $3", - vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await?; - let runs = rows - .into_iter() - .map(decode_row) - .collect::, _>>()?; + let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; commit(transaction).await?; Ok(runs) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 6027f5ea35..7fd0c800da 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -4,6 +4,7 @@ pub mod claim_next_run; pub(crate) mod frozen_job_request; pub mod heartbeat_run; pub(crate) mod job_write_set; +pub mod list_person_runs; pub mod list_run_claims; pub mod list_run_nodes; pub mod list_runs; @@ -13,5 +14,6 @@ pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; pub mod request_job_run; +pub mod request_person_job_run; pub mod request_rerun; pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs index 8593e77c41..35d9f616f0 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -1,13 +1,8 @@ //! Direct-store owners queue published jobs without supplying or widening their write scope. -use serde_json::json; -use uuid::Uuid; - -use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; -use crate::runs::main::frozen_job_request::frozen_job_request; -use crate::runs::models::{JobRunRequest, RunWindow}; +use crate::runs::_helpers::requests::manual_job::request_job_run_in; +use crate::runs::models::JobRunRequest; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -19,48 +14,12 @@ pub async fn request_job_run( context: &AuditContext, request: &JobRunRequest, ) -> Result { - if context.actor.trim().is_empty() - || context.granting_role.trim().is_empty() - || request.target.trim().is_empty() - || request.job_key.trim().is_empty() - || request.reason.trim().is_empty() - { - return Err(ControlError::Invalid( - "job run needs an actor, granting role, target, job and reason".into(), - )); - } - if request.window.as_ref().is_some_and(|window| match window { - RunWindow::Timestamp { start, end } => start > end, - RunWindow::Integer { start, end } => start > end, - }) { - return Err(ControlError::Invalid( - "backfill window start must not follow end".into(), - )); - } - let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); - let transaction = begin(store, TransactionMode::Write).await?; - let (transaction, retry) = request_retry( - transaction, - context, - (request.run_id, event_id, &json!(request)), + let (transaction, applied) = request_job_run_in( + begin(store, TransactionMode::Write).await?, + (context, None), + request, ) .await?; - if retry { - commit(transaction).await?; - return Ok(false); - } - let (transaction, frozen) = frozen_job_request(transaction, context, request).await?; - let (transaction, _) = publish_request(transaction, &frozen).await?; - let transaction = append_audit( - transaction, - context, - (event_id, "job.run_requested"), - ( - json!({"location_id": request.location_id, "target": request.target, "job_key": request.job_key, "run_id": request.run_id}), - serde_json::Value::Null, - json!({"request": request, "frozen": frozen}), - ), - ).await?; commit(transaction).await?; - Ok(true) + Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs new file mode 100644 index 0000000000..e8a33f2b2f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs @@ -0,0 +1,23 @@ +//! Browser callers cannot take the trusted local-owner path or supply an execution write set. + +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::manual_job::request_job_run_in; +use crate::runs::models::JobRunRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn request_person_job_run( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + request: &JobRunRequest, +) -> Result { + let (transaction, applied) = request_job_run_in( + begin(store, TransactionMode::Write).await?, + (context, Some(credentials)), + request, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 16625f5cc1..d63a994176 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -115,7 +115,7 @@ pub struct RunRequest { } /// A trusted operator names a published job; execution inputs are never supplied by the caller. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct JobRunRequest { pub run_id: Uuid, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index cc627ae85b..4ec73f9146 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -22,6 +22,8 @@ pub(crate) const PASSWORD_CREDENTIALS: &str = include_str!("../password_credenti pub(crate) const PASSWORD_SESSIONS: &str = include_str!("../password_sessions.sql"); pub(crate) const AUTHENTICATION_REVISION: &str = include_str!("../authentication_revision.sql"); pub(crate) const OIDC_LOGIN_FLOWS: &str = include_str!("../oidc_login_flows.sql"); +pub(crate) const ADMINISTRATOR_INITIALIZATION: &str = + include_str!("../administrator_initialization.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/administrator_initialization.sql b/crates/sqlbuild-orchestration/src/store/administrator_initialization.sql new file mode 100644 index 0000000000..f490c2402f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/administrator_initialization.sql @@ -0,0 +1,19 @@ +CREATE TABLE sqb_admin_initialization ( + singleton BIGINT PRIMARY KEY CHECK (singleton=1), + principal_id {uuid} NOT NULL, + initialized_at {time} NOT NULL +); +CREATE TABLE sqb_role_assignments ( + assignment_id {uuid} PRIMARY KEY, + principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), + role_key TEXT NOT NULL, + location_id {uuid} REFERENCES sqb_locations(location_id), + target TEXT, + source TEXT NOT NULL, + granted_by TEXT NOT NULL, + granted_at {time} NOT NULL, + revoked_by TEXT, + revoked_at {time}, + active_key TEXT UNIQUE +); +CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); diff --git a/crates/sqlbuild-orchestration/src/store/errors.rs b/crates/sqlbuild-orchestration/src/store/errors.rs index 54a8a6d29d..41af8db6fc 100644 --- a/crates/sqlbuild-orchestration/src/store/errors.rs +++ b/crates/sqlbuild-orchestration/src/store/errors.rs @@ -13,6 +13,11 @@ pub enum ControlError { Io(#[from] std::io::Error), #[error("invalid control-plane data: {0}")] Invalid(String), + #[error("control-plane operation denied: {reason}; decision {decision_id}")] + Denied { + reason: String, + decision_id: uuid::Uuid, + }, #[error("control-store schema is unsupported: {0}")] Schema(String), #[error("snapshot generation is not newer than the current generation")] diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index e63bfc3c31..4fe30caac3 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,10 +4,10 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, LOCATION_INSTANCES, - LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, - PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, render, verify_migrations, + ADMINISTRATOR_INITIALIZATION, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, + LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, + PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, + WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -46,6 +46,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("password_sessions", PASSWORD_SESSIONS), ("authentication_revision", AUTHENTICATION_REVISION), ("oidc_login_flows", OIDC_LOGIN_FLOWS), + ("administrator_initialization", ADMINISTRATOR_INITIALIZATION), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 9796d4536c..08981f832a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -49,7 +49,8 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable for (version, location_cleanup) in [(3, ""), (2, "DROP TABLE sqb_location_instances;")] { execute(store, &format!( - "DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ + "DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ + DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; \ DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 203aebd551..8f6a746dc1 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -23,6 +23,8 @@ pub(crate) async fn fingerprint_history( // Upgrade the immediate predecessor while published snapshots and freshness facts exist. let mut transaction = begin(store, TransactionMode::Write).await.expect("upgrade"); for table in [ + "sqb_role_assignments", + "sqb_admin_initialization", "sqb_oidc_login_flows", "sqb_authentication_revision", "sqb_password_sessions", @@ -46,7 +48,7 @@ pub(crate) async fn fingerprint_history( .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows','administrator_initialization')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs new file mode 100644 index 0000000000..34d6000912 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs @@ -0,0 +1,390 @@ +//! Installation bootstrap and final-admin transitions serialize across real database replicas. + +use futures_util::future::join_all; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{ + bootstrap_user::bootstrap_user, change_administrator::change_administrator, +}; +use crate::credentials::main::{ + create_principal::create_principal, login_password::login_password, + revoke_session::revoke_session, set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorChange, AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind, +}; +use crate::credentials::tests::helpers::authentication_revision; +use crate::credentials::tests::test_types::AdministratorTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_only_and_final_admin_survives() + { + let test_cases = [ + AdministratorTestCase { + description: "SQLite once-only bootstrap and final admin", + backend: Backend::Sqlite, + expected_administrators: 1, + }, + AdministratorTestCase { + description: "PostgreSQL once-only bootstrap and final admin", + backend: Backend::Postgres, + expected_administrators: 1, + }, + ]; + for test_case in test_cases { + for change in [AdministratorChange::Revoke, AdministratorChange::Disable] { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted peer")), + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json"); + let inputs: Value = serde_json::from_str( + &std::fs::read_to_string(path).expect("DuckShop lifecycle"), + ) + .expect("inputs"); + let names = [ + inputs["first_person"].as_str().expect("first person"), + inputs["second_person"].as_str().expect("second person"), + ]; + let stores = [store, &replica]; + let results = join_all( + (0..2).map(|index| bootstrap_user(stores[index], &context, names[index])), + ) + .await; + assert_eq!( + results.iter().filter(|result| result.is_ok()).count(), + test_case.expected_administrators, + "{}", + test_case.description + ); + let (winner, first) = results + .into_iter() + .enumerate() + .find_map(|(index, result)| result.ok().map(|id| (index, id))) + .expect("one bootstrap winner"); + let other = 1 - winner; + let second = + create_principal(store, &context, (names[other], PrincipalKind::Person)) + .await + .expect("ordinary role-free person"); + let password = inputs["password"].as_str().expect("password input"); + let mut sessions = Vec::new(); + for (id, name) in [(first, names[winner]), (second, names[other])] { + set_password(store, &context, PasswordAuthority::Operator(id), password) + .await + .expect("owner provisioning"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, password)) + .await + .expect("login") + else { + panic!("valid login"); + }; + sessions.push(session); + } + let first_credentials = (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()); + let second_credentials = (sessions[1].bearer.as_str(), sessions[1].csrf.as_str()); + assert!( + change_administrator( + store, + &context, + second_credentials, + (second, AdministratorChange::Grant) + ) + .await + .is_err(), + "a valid person session is not administrator authority" + ); + assert!( + change_administrator( + store, + &context, + (first_credentials.0, "wrong-csrf"), + (second, AdministratorChange::Grant) + ) + .await + .is_err() + ); + assert!( + change_administrator(store, &context, first_credentials, (first, change)) + .await + .is_err(), + "cannot remove sole administrator" + ); + assert!( + change_administrator( + store, + &context, + first_credentials, + (second, AdministratorChange::Grant) + ) + .await + .expect("authenticated administrator grant") + ); + let revision = authentication_revision(store) + .await + .expect("revision before duplicate grant"); + assert!( + !change_administrator( + store, + &context, + first_credentials, + (second, AdministratorChange::Grant) + ) + .await + .expect("idempotent grant") + ); + assert_eq!( + authentication_revision(store) + .await + .expect("revision after duplicate grant"), + revision + ); + let (left, right) = tokio::join!( + change_administrator(store, &context, first_credentials, (first, change)), + change_administrator(&replica, &context, second_credentials, (second, change)) + ); + assert_eq!( + [&left, &right] + .iter() + .filter(|result| matches!(result, Ok(true))) + .count(), + 1, + "concurrent self-removal leaves one administrator" + ); + let survivor = usize::from(left.is_ok()); + let survivor_credentials = ( + sessions[survivor].bearer.as_str(), + sessions[survivor].csrf.as_str(), + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("inspect remaining admins"), + "SELECT COUNT(*) AS count FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) WHERE a.role_key='admin' AND a.revoked_at IS NULL AND p.disabled_at IS NULL",vec![]).await.expect("count remaining admins"); + assert_eq!(rows[0]["count"], json!(1)); + commit(transaction) + .await + .expect("finish remaining-admin read"); + let (transaction, audit) = query(begin(store,TransactionMode::Read).await.expect("read grant audit"), + "SELECT actor,granting_role FROM sqb_security_audit WHERE action='role.assigned' AND actor<> 'local-control-store-owner'",vec![]).await.expect("grant audit"); + assert_eq!(audit.len(), 1); + assert_eq!(audit[0]["actor"], json!(format!("principal:{first}"))); + assert_eq!(audit[0]["granting_role"], json!("admin")); + commit(transaction).await.expect("finish audit read"); + revoke_session(store, &context, &sessions[survivor].identity) + .await + .expect("logout"); + assert!( + change_administrator( + &replica, + &context, + survivor_credentials, + (first, AdministratorChange::Grant) + ) + .await + .is_err(), + "revoked session cannot grant" + ); + // Simulate an operator's out-of-band loss: it must never reopen bootstrap. + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("simulate out-of-band loss"), + "UPDATE sqb_principals SET disabled_at=$1", + vec![Parameter::Time(chrono::Utc::now())], + ) + .await + .expect("disable all people outside supported API"); + commit(transaction).await.expect("commit simulated loss"); + assert!( + bootstrap_user(&replica, &context, "replacement-admin") + .await + .is_err() + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_authority_rolls_back() +{ + let test_cases = [ + AdministratorTestCase { + description: "SQLite publication failures roll back authority", + backend: Backend::Sqlite, + expected_administrators: 0, + }, + AdministratorTestCase { + description: "PostgreSQL publication failures roll back authority", + backend: Backend::Postgres, + expected_administrators: 0, + }, + ]; + for test_case in test_cases { + for (table, operation) in [ + ("sqb_security_audit", "INSERT"), + ("sqb_authentication_revision", "UPDATE"), + ] { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted peer")), + }; + let fault = TransactionFault { + description: "fail administrator publication", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'publication unavailable')", + postgres_body: "RAISE EXCEPTION 'publication unavailable'", + }; + install_fault(store, &fault).await; + assert!( + bootstrap_user(store, &context, "orders-admin") + .await + .is_err() + ); + remove_fault(store, &fault).await; + let (transaction,rows) = query(begin(store,TransactionMode::Read).await.expect("inspect rollback"),"SELECT (SELECT COUNT(*) FROM sqb_principals) AS people, (SELECT COUNT(*) FROM sqb_admin_initialization) AS initialized, (SELECT COUNT(*) FROM sqb_role_assignments) AS assignments",vec![]).await.expect("count rolled back bootstrap"); + assert_eq!( + serde_json::to_value(&rows[0]).expect("serializable row"), + json!({"people":0,"initialized":0,"assignments":0}) + ); + assert_eq!( + rows[0]["assignments"], + json!(test_case.expected_administrators), + "{}", + test_case.description + ); + commit(transaction).await.expect("finish rollback read"); + assert_eq!( + authentication_revision(store) + .await + .expect("rolled back revision"), + 0 + ); + let first = bootstrap_user(store, &context, "orders-admin") + .await + .expect("retry after rollback"); + let second = + create_principal(store, &context, ("customers-admin", PrincipalKind::Person)) + .await + .expect("second person"); + set_password( + store, + &context, + PasswordAuthority::Operator(first), + "orders-password-2026", + ) + .await + .expect("owner password provisioning"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, ("orders-admin", "orders-password-2026")) + .await + .expect("password login") + else { + panic!("login"); + }; + let credentials = (session.bearer.as_str(), session.csrf.as_str()); + let revision = authentication_revision(store) + .await + .expect("revision before failed grant"); + install_fault(store, &fault).await; + assert!( + change_administrator( + store, + &context, + credentials, + (second, AdministratorChange::Grant) + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + authentication_revision(store) + .await + .expect("revision after failed grant"), + revision + ); + assert!( + change_administrator( + store, + &context, + credentials, + (second, AdministratorChange::Grant) + ) + .await + .expect("retry grant"), + "failed grant did not persist" + ); + assert!( + change_administrator( + store, + &context, + credentials, + (first, AdministratorChange::Disable) + ) + .await + .is_err(), + "an unprovisioned administrator is not a usable password-authentication alternative" + ); + set_password( + store, + &context, + PasswordAuthority::Operator(second), + "customers-password-2026", + ) + .await + .expect("provision alternative administrator"); + install_fault(store, &fault).await; + assert!( + change_administrator( + store, + &context, + credentials, + (first, AdministratorChange::Disable) + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + change_administrator( + store, + &context, + credentials, + (first, AdministratorChange::Disable) + ) + .await + .expect("retry disabling person"), + "failed disable preserved live session and enabled person" + ); + close_test_store(fixture).await; + }); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index 7bf01dda3a..debcc4d815 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -4,6 +4,8 @@ pub(crate) mod helpers; #[path = "locations/helpers.rs"] pub(crate) mod location_helpers; +#[path = "test_administrators.rs"] +mod test_administrators; #[path = "test_contract.rs"] mod test_contract; #[path = "passwords/test_contract.rs"] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index c2efe4bf2a..363d89bd7d 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -6,3 +6,9 @@ pub(crate) struct CredentialTestCase { pub(crate) backend: Backend, pub(crate) expected_hash_length: usize, } + +pub(crate) struct AdministratorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_administrators: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index 530eb54c35..d4350c8344 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -20,6 +20,8 @@ mod test_groups; mod test_lifecycle; #[path = "test_monitor.rs"] mod test_monitor; +#[path = "test_permissions.rs"] +mod test_permissions; #[path = "test_requests.rs"] mod test_requests; #[path = "test_reruns.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs new file mode 100644 index 0000000000..d1f32fb678 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -0,0 +1,261 @@ +//! Real session, role, immutable-job and database paths enforce sensitive-run permissions. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, + change_administrator::change_administrator, +}; +use crate::credentials::main::{ + create_principal::create_principal, login_password::login_password, set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorChange, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, + PrincipalKind, RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + list_person_runs::list_person_runs, list_runs::list_runs, + request_person_job_run::request_person_job_run, +}; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::PersonRunPermissionTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_administrators_can_rewrite_history() + { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json"); + let inputs: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop person lifecycle")) + .expect("lifecycle settings"); + let settings_count = inputs["job_settings"].as_array().expect("settings").len(); + let test_cases = [ + PersonRunPermissionTestCase { + description: "viewer cannot run ordinary jobs", + role: BuiltinRole::Viewer, + settings_range: 0..1, + expected_allowed: false, + }, + PersonRunPermissionTestCase { + description: "operator can run ordinary jobs", + role: BuiltinRole::Operator, + settings_range: 0..1, + expected_allowed: true, + }, + PersonRunPermissionTestCase { + description: "developer can run ordinary jobs", + role: BuiltinRole::Developer, + settings_range: 0..1, + expected_allowed: true, + }, + PersonRunPermissionTestCase { + description: "operator cannot rewrite history", + role: BuiltinRole::Operator, + settings_range: 1..settings_count, + expected_allowed: false, + }, + PersonRunPermissionTestCase { + description: "developer cannot rewrite history", + role: BuiltinRole::Developer, + settings_range: 1..settings_count, + expected_allowed: false, + }, + PersonRunPermissionTestCase { + description: "installation administrator can rewrite history", + role: BuiltinRole::Admin, + settings_range: 1..settings_count, + expected_allowed: true, + }, + ]; + for backend in [Backend::Sqlite, Backend::Postgres] { + for test_case in &test_cases { + for settings_index in test_case.settings_range.clone() { + run(async { + let fixture = open_test_store(backend).await.expect("real database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted peer")), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = + create_principal(store, &context, ("orders-person", PrincipalKind::Person)) + .await + .expect("role-free person"); + let mut sessions = Vec::new(); + for (id, name) in [(admin, "orders-admin"), (person, "orders-person")] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("provision password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid session") + }; + sessions.push(session); + } + let credentials = (sessions[1].bearer.as_str(), sessions[1].csrf.as_str()); + let location_id = create_location(store, "orders", "test-owner") + .await + .expect("location"); + let mut input = snapshot(location_id); + input.documents.insert("jobs".into(), json!({ + "daily_orders": {"description":"Build orders", "selection":["orders"], "settings":inputs["job_settings"][settings_index]}, + "customers": {"description":"Build customers", "selection":["customers"], "settings":inputs["job_settings"][0]} + })); + input.documents.insert("groups".into(), json!({"orders_group": {"description":"Build both jobs", "selection":["customers","orders"], "jobs":{"daily_orders":[],"customers":[]}}})); + publish_snapshot(store, &input) + .await + .expect("publish immutable job"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh order totals".into(), + priority: 0, + window: None, + }; + assert!( + matches!( + request_person_job_run(store, (&context, credentials), &request).await, + Err(ControlError::Denied { .. }) + ), + "no role is the default" + ); + let assignment = RoleAssignmentRequest { + principal_id: person, + role: test_case.role, + location_id: None, + target: None, + revoke: false, + }; + assert!( + assign_builtin_role( + store, + ( + &context, + (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()) + ), + &assignment + ) + .await + .expect("authenticated assignment") + ); + assert!( + list_person_runs( + store, + (&context, credentials), + (location_id, "development", 10) + ) + .await + .expect("built-in metadata access") + .is_empty() + ); + for job_key in ["daily_orders", "orders_group"] { + let result = request_person_job_run( + store, + (&context, credentials), + &JobRunRequest { + run_id: Uuid::now_v7(), + job_key: job_key.into(), + ..request.clone() + }, + ) + .await; + assert_eq!( + matches!(&result, Err(ControlError::Denied { .. })), + !test_case.expected_allowed, + "{}: {result:?}", + test_case.description + ); + assert_eq!( + result.ok(), + test_case.expected_allowed.then_some(true), + "{}", + test_case.description + ); + } + assert_eq!( + list_runs(store, (location_id, "development"), 10) + .await + .expect("queue projection") + .len(), + 4 * usize::from(test_case.expected_allowed), + "denial cannot publish work; success queues the job, group and two children" + ); + let administrator_credentials = + (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()); + assert!( + change_administrator( + store, + &context, + administrator_credentials, + (person, AdministratorChange::Disable) + ) + .await + .expect("disable person") + ); + assert!( + assign_builtin_role( + store, + (&context, administrator_credentials), + &RoleAssignmentRequest { + revoke: true, + ..assignment + } + ) + .await + .expect("revoke a disabled person's grant before reenabling") + ); + assert!( + change_administrator( + store, + &context, + administrator_credentials, + (person, AdministratorChange::Enable) + ) + .await + .expect("enable person") + ); + assert!( + matches!( + request_person_job_run( + store, + (&context, credentials), + &JobRunRequest { + run_id: Uuid::now_v7(), + ..request + } + ) + .await, + Err(ControlError::Denied { .. }) + ), + "reenabling cannot restore a revoked grant" + ); + close_test_store(fixture).await; + }); + } + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index fc108dd8c5..078f2c0a62 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -2,11 +2,19 @@ use std::collections::BTreeSet; +use crate::credentials::models::BuiltinRole; use crate::runs::models::{RerunRequest, RunStatus}; pub(crate) use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_test_types::Backend; +pub(crate) struct PersonRunPermissionTestCase { + pub(crate) description: &'static str, + pub(crate) role: BuiltinRole, + pub(crate) settings_range: std::ops::Range, + pub(crate) expected_allowed: bool, +} + pub(crate) struct ScopedClaimTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs index 3cc7469c46..926ad7363e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -15,6 +15,7 @@ use crate::runs::main::{ use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, }; +use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::{ begin_sensor_evaluation::begin_sensor_evaluation, @@ -132,23 +133,30 @@ fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_mu outcome: RunOutcome::Success, reason: "All logical nodes succeed".into(), exit_code: None, - })); - for change in changes { - change_run( - store, - RunAuthority::Worker { - run_id: claim.request.run_id, - claim_id: claim.claim_id, - }, - &RunChangeRequest { - event_id: Uuid::now_v7(), - run_id: claim.request.run_id, - claim_id: Some(claim.claim_id), - change, - }, - ) - .await - .expect("real node evidence"); + })) + .collect::>(); + for batch in changes.chunks(100) { + // Evidence setup must keep its scheduler alive, just like the real loop. + renew_lease(store, &lease) + .await + .expect("renew during setup"); + for change in batch { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: change.clone(), + }, + ) + .await + .expect("real node evidence"); + } } advance_groups(store, &lease, (&limits, scope)) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index cfb8c1e19f..2cc01e3a1c 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -426,6 +426,8 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .await .expect("upgrade fixture"); for table in [ + "sqb_role_assignments", + "sqb_admin_initialization", "sqb_oidc_login_flows", "sqb_authentication_revision", "sqb_password_sessions", diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 526a406cd9..c2d836b808 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,8 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ + "DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ + DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ DROP INDEX sqb_node_decisive_history; \ diff --git a/deploy/README.md b/deploy/README.md index 49fa889686..a4bef43bfd 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -116,8 +116,33 @@ For opt-in OIDC identity verification, mount a server-owned JSON file with `orig `--oidc-config /etc/sqlbuild/oidc.json` to the server command. Both origin and issuer require HTTPS. Terminate HTTPS at a trusted proxy, preserve the external Host, restrict backend access, and register `/api/auth/oidc/callback` at the issuer. This verifies an explicitly bound enabled -person; it does not provision people, issue a browser session, or grant roles. The reference -does not assign an initial administrator or authorize non-admin operations. +person; it does not provision people, issue a browser session, or grant roles. + +The local control-store owner initializes the first installation administrator with +`sqb admin create-user NAME`, then provisions its password separately with +`sqb admin set-password PRINCIPAL_ID`. Initialization is available once only; losing or disabling +administrators never reopens it. `create-principal` creates a role-free identity, not another +administrator. With password authentication enabled, authenticated installation administrators +use `POST /api/admin/people/{principal_id}` with a JSON action of `grant`, `revoke`, `disable`, +or `enable`, their session cookie and CSRF header. This cannot remove or disable the final enabled, +password-provisioned administrator; an unprovisioned person is not a usable alternative in this +password-only administration path. It never authorizes a machine bearer or grants non-admin defaults. +The explicit offline recovery command is not yet available. + +An installation administrator assigns or revokes additive built-in roles through +`POST /api/admin/roles`, using `principal_id`, `role`, optional `location_id` and `target`, +and `revoke` (false for a grant). An administrator assignment must be installation-wide. +Other built-ins can be installation-wide or restricted to a location or target. Revocation +also works while a person is disabled, before they are enabled again. + +Password sessions use `GET /api/control/locations/{location_id}/targets/{target}/runs` +for recent run metadata, `POST /api/control/runs` for published-job requests, and +`POST /api/control/triggers` for trigger overrides. Viewers can read metadata but cannot +run jobs; operators and developers can run ordinary jobs and toggle triggers. Full-refresh +jobs, published cursor bounds, explicit backfill windows and cursor resets require an installation +administrator in these endpoints. A person with no assigned role cannot use them. Metadata responses do not +contain SQL, sampled rows or captured output. These person routes do not accept machine +credentials; location and worker credentials continue to use the separate machine routes. Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in `.env` before creating the server to receive the native platform stream. Leave it unset to disable delivery. Receivers must deduplicate by `event_id` diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a1386b584f..2a4c03f0a5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -238,7 +238,9 @@ def api_address(process: subprocess.Popen[bytes], log: Path) -> str: @contextmanager -def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: +def snapshot_api( + directory: Path, *, server_arguments: tuple[str, ...] = () +) -> Iterator[SnapshotApi]: """Only this fixture seeds the control database; the code location uses HTTP exclusively.""" directory.mkdir() log: Path = directory / "server.log" @@ -247,7 +249,14 @@ def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) with log.open("wb") as stderr: process: subprocess.Popen[bytes] = subprocess.Popen( - [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", "--no-scheduler"], + [ + str(Path(sys.executable).with_name("sqb")), + "server", + "--port", + "0", + "--no-scheduler", + *server_arguments, + ], cwd=directory, env=environment, stdout=subprocess.DEVNULL, @@ -331,7 +340,11 @@ def uncooperative_launcher(directory: Path, executable: str) -> Iterator[WorkerD @contextmanager def scheduled_worker( - directory: Path, monkeypatch: pytest.MonkeyPatch, test_case: WorkerExecutionTestCase + directory: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: WorkerExecutionTestCase, + *, + server_arguments: tuple[str, ...] = (), ) -> Iterator[ScheduledWorker]: """Declare, publish and schedule isolated DuckDB work without provisioning claims.""" project: Path = directory / "project" @@ -365,7 +378,7 @@ def scheduled_worker( "uncooperative": uncooperative_launcher, } with ( - snapshot_api(directory / "control") as api, + snapshot_api(directory / "control", server_arguments=server_arguments) as api, launchers[test_case.launcher](directory, executable) as deployment, ): published: subprocess.CompletedProcess[str] = subprocess.run( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py new file mode 100644 index 0000000000..34fd0c62ac --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -0,0 +1,84 @@ +"""Exercise owner provisioning and browser transport without bypassing role authorization.""" + +import json +import os +import subprocess +import sys +from http.cookies import SimpleCookie +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotApi, +) + + +def provision_person(control: Path, command: str, name: str) -> dict[str, Any]: + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + executable: str = str(Path(sys.executable).with_name("sqb")) + created: subprocess.CompletedProcess[str] = subprocess.run( + [executable, "admin", command, name], + cwd=control, + env=environment, + capture_output=True, + text=True, + check=True, + timeout=15, + ) + person: dict[str, Any] = json.loads(created.stdout) + _ = subprocess.run( + [executable, "admin", "set-password", person["principal_id"]], + input="orders-person-password-2026\n", + cwd=control, + env=environment, + capture_output=True, + text=True, + check=True, + timeout=15, + ) + return person + + +def person_headers(session: tuple[str, str]) -> dict[str, str]: + return { + "Host": "control.example.com", + "Origin": "https://control.example.com", + "Content-Type": "application/json", + "Cookie": session[0], + "x-sqb-csrf": session[1], + } + + +def person_read(api: SnapshotApi, path: str, *, session: tuple[str, str]) -> Any: + with urlopen(Request(api.url + path, headers=person_headers(session)), timeout=5) as response: + return json.load(response) + + +def person_request( + api: SnapshotApi, path: str, *, body: Any, session: tuple[str, str] = ("", "") +) -> Any: + request: Request = Request( + api.url + path, + data=json.dumps(body).encode(), + headers=person_headers(session), + ) + with urlopen(request, timeout=5) as response: + return json.load(response) + + +def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: + request: Request = Request( + api.url + "/api/auth/password/login", + data=json.dumps({"username": name, "password": "orders-person-password-2026"}).encode(), + headers={ + "Host": "control.example.com", + "Origin": "https://control.example.com", + "Content-Type": "application/json", + }, + ) + with urlopen(request, timeout=5) as response: + cookies: SimpleCookie[str] = SimpleCookie("\n".join(response.headers.get_all("Set-Cookie"))) + cookie: str = "__Host-sqb-session=" + cookies["__Host-sqb-session"].value + return cookie, json.load(response)["csrf_token"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py new file mode 100644 index 0000000000..ce3aabb453 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -0,0 +1,205 @@ +"""A real owner, browser, scheduler, location and DuckDB share one permission-enforced journey.""" + +import json +import sys +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + scheduled_worker, + wait_for_generation, + wait_for_location_health, + wait_for_run, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "owner bootstrap, least-privilege browser operations, reload and last-good recovery", + "SELECT 23 AS amount", + "SELECT amount FROM orders", + "success", + [(23,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_an_initialized_installation_when_people_run_and_reload_then_permissions_and_recovery_hold( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + scheduled: dict[str, Any] = worker_result(worker, test_case) + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-person" + ) + admin_session: tuple[str, str] = person_login(worker.api, admin["name"]) + session: tuple[str, str] = person_login(worker.api, person["name"]) + runs_path: str = f"/api/control/locations/{worker.api.location_id}/targets/development/runs" + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": worker.api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Refresh order totals", + "priority": 0, + } + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "viewer", + "location_id": worker.api.location_id, + "target": "development", + } + with pytest.raises(HTTPError) as no_role: + person_read(worker.api, runs_path, session=session) + assert no_role.value.code == 403 + for path, body in [ + ("/api/control/runs", request), + ("/api/admin/roles", assignment), + ]: + with pytest.raises(HTTPError) as rejected: + person_request(worker.api, path, body=body, session=session) + assert rejected.value.code == 403 + assert "decision" in json.load(rejected.value)["error"] + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + metadata: list[dict[str, Any]] = person_read(worker.api, runs_path, session=session) + assert metadata[0]["run_id"] == scheduled["run_id"] + assert not ({"sql", "compiled_sql", "output", "rows", "evidence"} & metadata[0].keys()) + for path, body in [ + ("/api/control/runs", request), + ("/api/admin/people/" + admin["principal_id"], "disable"), + ]: + with pytest.raises(HTTPError) as rejected: + person_request(worker.api, path, body=body, session=session) + assert rejected.value.code == 403 + with pytest.raises(HTTPError) as wrong_target: + person_read(worker.api, runs_path.replace("development", "preview"), session=session) + assert wrong_target.value.code == 403 + assignment["role"] = "operator" + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as csrf_rejected: + person_request( + worker.api, "/api/control/runs", body=request, session=(session[0], "wrong-csrf") + ) + assert csrf_rejected.value.code == 401 + for sensitive in [request | {"window": {"kind": "integer", "start": 3, "end": 8}}]: + with pytest.raises(HTTPError) as rewrite_rejected: + person_request(worker.api, "/api/control/runs", body=sensitive, session=session) + assert rewrite_rejected.value.code == 403 + assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { + "applied": True + } + manual: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=request["run_id"] + ) + assert manual["status"] == test_case.expected_status + assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { + "applied": False + } + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + ) as location: + _ = wait_for_generation(worker.api, "development", 2, location, log) + model: Path = worker.project / "models/orders.sql" + original: str = model.read_text() + _ = model.write_text(original.replace("23", "37")) + edited: dict[str, Any] = wait_for_generation( + worker.api, "development", 3, location, log + ) + request["run_id"] = str(uuid4()) + assert person_request( + worker.api, "/api/control/runs", body=request, session=session + ) == {"applied": True} + reloaded: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=request["run_id"] + ) + assert reloaded["status"] == test_case.expected_status + assert reloaded["snapshot_id"] == edited["record"]["snapshot_id"] + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(37,)] + _ = model.write_text("SELECT FROM invalid SQL !\n") + _ = wait_for_location_health( + worker.api, + "development", + {"kind": "compile_failed", "generation": 4}, + location, + log, + ) + assert current_snapshot(worker.api, "development")["record"] == edited["record"] + _ = model.write_text(original.replace("23", "41")) + recovered: dict[str, Any] = wait_for_generation( + worker.api, "development", 4, location, log + ) + request["run_id"] = str(uuid4()) + assert person_request( + worker.api, "/api/control/runs", body=request, session=session + ) == {"applied": True} + recovery: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=request["run_id"] + ) + assert recovery["status"] == test_case.expected_status + assert recovery["snapshot_id"] == recovered["record"]["snapshot_id"] + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(41,)] + assignment["revoke"] = True + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as revoked: + person_request( + worker.api, + "/api/control/runs", + body=request | {"run_id": str(uuid4())}, + session=session, + ) + assert revoked.value.code == 403 From d1657fb5d26f36e113439a6c94c9db99f6eab2fd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:24:05 +0000 Subject: [PATCH 079/138] feat: add exclusive offline administrator recovery --- Cargo.lock | 11 + Cargo.toml | 1 + .../administrator_lifecycle.json | 1 + .../sqlbuild-cli/src/argv/command_table.json | 13 + .../src/native_commands/_helpers/execution.rs | 37 +- .../main/run_control_invocation.rs | 4 +- .../native_commands/admin/test_passwords.rs | 33 ++ .../tests/native_commands/helpers.rs | 27 +- crates/sqlbuild-orchestration/Cargo.toml | 2 + .../src/authorization/main/mod.rs | 1 + .../main/recover_administrator.rs | 71 ++++ .../src/store/_helpers/mod.rs | 1 + .../src/store/_helpers/open_control_store.rs | 39 +++ .../src/store/_helpers/open_sqlite.rs | 42 ++- .../src/store/main/begin.rs | 10 +- .../src/store/main/begin_offline_recovery.rs | 26 ++ .../src/store/main/commit.rs | 5 +- .../src/store/main/mod.rs | 1 + .../src/store/main/open_store.rs | 19 +- .../src/store/main/query.rs | 7 +- .../src/store/main/subscribe_changes.rs | 2 +- .../src/store/models.rs | 9 +- .../tests/api/changes/helpers.rs | 2 +- .../tests/api/output/helpers.rs | 2 +- .../tests/api/work/helpers.rs | 2 +- .../tests/credentials/helpers.rs | 4 +- .../tests/credentials/test_contracts.rs | 2 + .../tests/credentials/test_recovery.rs | 328 ++++++++++++++++++ .../tests/credentials/test_types.rs | 6 + .../tests/runs/claims/helpers.rs | 2 +- .../tests/runs/monitor/helpers.rs | 2 +- .../tests/scheduler/ticks/helpers.rs | 2 +- .../tests/snapshots/helpers.rs | 2 +- .../tests/store/faults/helpers.rs | 4 +- .../tests/store/leases/helpers.rs | 2 +- .../tests/store/replicas/helpers.rs | 24 +- deploy/README.md | 12 +- 37 files changed, 691 insertions(+), 67 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/open_control_store.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/begin_offline_recovery.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs diff --git a/Cargo.lock b/Cargo.lock index 228f904fae..03978b66b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1183,6 +1183,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "fs4" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e72ed92b67c146290f88e9c89d60ca163ea417a446f61ffd7b72df3e7f1dfd5" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -3723,6 +3733,7 @@ dependencies = [ "chrono-tz", "croner", "flate2", + "fs4", "futures-util", "getrandom 0.4.3", "openidconnect", diff --git a/Cargo.toml b/Cargo.toml index 652b497017..50c8bcca7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ argon2 = "0.5.3" axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } axum-server = { version = "0.8", default-features = false, features = ["tls-rustls-no-provider"] } fensu-policy = "=0.22.0" +fs4 = "1.1" futures-util = { version = "0.3", default-features = false, features = ["std"] } # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json index 212ca0c21c..db7faa5392 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -2,6 +2,7 @@ "first_person": "orders-admin", "second_person": "customers-admin", "password": "orders-bootstrap-password-2026", + "recovery_reason": "restore access after loss of the final administrator", "disabled_person_sequence": ["assign", "disable", "revoke", "enable", "deny_run"], "job_settings": [ {"full_refresh": false}, diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index b5fd08acbf..f781b02f9d 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -352,6 +352,19 @@ ] } }, + { + "name": "recover-user", + "parser": { + "prog": "sqb admin recover-user", + "description": "Recover an existing installation administrator offline through local control-store ownership. Stop every server and administration process first. Requires exclusive access, records the reason durably and revokes the person's sessions; never reopens bootstrap. Password provisioning is separate.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "PRINCIPAL_ID", "required": true, "help": "existing person UUID to enable and restore as installation admin", "group": 0}, + {"kind": "store", "options": ["--reason"], "dest": "admin_reason", "required": true, "help": "durable explanation for offline recovery", "group": 1} + ] + } + }, { "name": "issue-location-token", "parser": { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index b36264baaf..6036e52bf5 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -10,6 +10,8 @@ use chrono::{DateTime, Utc}; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, DockerImageSource}; +use sqlbuild_orchestration::authorization::main::recover_administrator::recover_administrator; +use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; @@ -20,6 +22,7 @@ use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::begin_offline_recovery::begin_offline_recovery; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; @@ -51,6 +54,7 @@ use crate::native_commands::constants::{ COLUMNS_VARIABLE, FALLBACK_COLUMNS, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, @@ -164,7 +168,7 @@ fn development_location_role( .env_remove("SQLBUILD_DEVELOPMENT_CONTROL_FILE") .stdin(Stdio::null()) .kill_on_drop(true); - if let ControlStore::Sqlite(path) = store { + if let ControlStore::Sqlite(path, _) = store { command.env( "SQLBUILD_DEVELOPMENT_CONTROL_FILE", std::env::current_dir()?.join(path), @@ -480,6 +484,37 @@ pub(crate) fn start_platform_roles( Ok(()) } +/// Keep exclusive offline recovery outside the online store-opening and migration path. +pub(crate) async fn run_administration( + namespace: &Namespace, +) -> Result { + if text(namespace, "admin_command").as_deref() != Some("recover-user") { + return run_admin(namespace).await; + } + let principal_id = Uuid::parse_str(&text(namespace, "admin_subject").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("a valid subject UUID is required".into()))?; + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if reason.trim().is_empty() { + return Err(ControlError::Invalid( + "offline recovery requires --reason".into(), + )); + } + let url = control_database(namespace)?; + drop(write_output(&[( + Stream::Stderr, + "Acquiring exclusive offline control-store access...\n", + )])); + let recovery = begin_offline_recovery(&url).await?; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + recover_administrator(recovery, &context, (principal_id, &reason)).await?; + Ok(serde_json::json!({"principal_id":principal_id,"role":"admin","recovered":true})) +} + /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. pub(crate) async fn open_control_store( namespace: &Namespace, diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs index 6ff78bf47b..d56e98354b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -4,9 +4,9 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::execution::run_administration; use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; -use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_control::run_control; pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { @@ -50,7 +50,7 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { let role = if admin { "administration" } else { "service" }; let result = runtime.block_on(async { if admin { - run_admin(namespace).await.map(Some) + run_administration(namespace).await.map(Some) } else { run_control(namespace, command == SCHEDULER_COMMAND) .await diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs index d48b3e763b..e52559cc74 100644 --- a/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs @@ -73,6 +73,39 @@ fn given_local_store_ownership_when_create_user_runs_then_only_the_first_process } let initialized: i64 = connection.query_row("SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.initialized' AND actor='local-control-store-owner'",[],|row| row.get(0)).expect("bootstrap audit"); assert_eq!(initialized,1); + let arguments = ["recover-user", id.as_str(), "--reason", "restore access after administrator loss"]; + let rejected = invoke_admin(&control, &arguments); + assert!(!rejected.status.success(), "online recovery is forbidden"); + assert!(rejected.stdout.is_empty()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("exclusive access")); + control.child.kill().expect("stop server before offline recovery"); + assert!(!wait_for_exit(&mut control).await.success()); + let recovery = invoke_admin(&control, &arguments); + assert!(recovery.status.success(), "{}", String::from_utf8_lossy(&recovery.stderr)); + assert_eq!(serde_json::from_slice::(&recovery.stdout).expect("JSON-only recovery"), json!({"principal_id":id,"role":"admin","recovered":true})); + assert_eq!(String::from_utf8_lossy(&recovery.stderr), "Acquiring exclusive offline control-store access...\nControl administration completed successfully\n"); + let (audits, initialized): (i64,i64) = connection.query_row("SELECT (SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.offline_recovered'),(SELECT COUNT(*) FROM sqb_admin_initialization)",[],|row| Ok((row.get(0)?,row.get(1)?))).expect("recovery and bootstrap audits"); + assert_eq!((audits,initialized),(1,1)); + control.child = crate::helpers::spawn_control((control.directory.path(), &control.stderr), &["server","--port","0","--no-scheduler","--password-origin","https://control.example.com"]); + let url = api_ready(&mut control).await; + let previous = client.get(format!("{url}/api/auth/session")).header(header::HOST,"control.example.com") + .header(header::COOKIE,&sessions[0].0).send().await.expect("old session after recovery"); + assert_eq!(previous.status(),StatusCode::UNAUTHORIZED); + let response = client.post(format!("{url}/api/auth/password/login")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .json(&json!({"username":first["name"],"password":"orders-admin-password-2026"})).send().await.expect("recovered admin login"); + assert_eq!(response.status(),StatusCode::OK); + let cookie = response.headers().get_all(header::SET_COOKIE).iter().filter_map(|value| value.to_str().ok()) + .find(|value| value.starts_with("__Host-sqb-session=")).expect("new session cookie").split(';').next().expect("cookie pair").to_owned(); + let body: Value = response.json().await.expect("recovered session"); + let denied = client.post(format!("{url}/api/admin/people/{id}")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .header(header::COOKIE,cookie).header("x-sqb-csrf",body["csrf_token"].as_str().expect("new CSRF")) + .json(&"revoke").send().await.expect("final-admin protection after recovery"); + assert_eq!(denied.status(),StatusCode::FORBIDDEN); + let rejected = invoke_admin(&control,&["create-user","replacement-admin"]); + assert!(!rejected.status.success()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("bootstrap is closed")); #[cfg(unix)] crate::helpers::terminate(&mut control).await; }); diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs index df025a6a36..7d3fe3ba71 100644 --- a/crates/sqlbuild-cli/tests/native_commands/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -19,17 +19,9 @@ pub(crate) fn start_control(arguments: &[&str]) -> TestControl { } pub(crate) fn start_control_in(directory: tempfile::TempDir, arguments: &[&str]) -> TestControl { - let binary = isolated_binary(directory.path()); + isolated_binary(directory.path()); let stderr = directory.path().join("control.log"); - let child = Command::new(binary) - .args(arguments) - .current_dir(directory.path()) - .env("PATH", "") - .env_remove("SQLBUILD_CONTROL_DATABASE_URL") - .stdout(Stdio::null()) - .stderr(std::fs::File::create(&stderr).expect("control stderr")) - .spawn() - .expect("start native control process"); + let child = spawn_control((directory.path(), &stderr), arguments); TestControl { child, directory, @@ -37,6 +29,21 @@ pub(crate) fn start_control_in(directory: tempfile::TempDir, arguments: &[&str]) } } +pub(crate) fn spawn_control( + (directory, stderr): (&Path, &Path), + arguments: &[&str], +) -> std::process::Child { + Command::new(directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX))) + .args(arguments) + .current_dir(directory) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout(Stdio::null()) + .stderr(std::fs::File::create(stderr).expect("control stderr")) + .spawn() + .expect("start native control process") +} + pub(crate) async fn api_ready(control: &mut TestControl) -> String { let client = reqwest::Client::builder() .timeout(Duration::from_secs(1)) diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index 6e944178d1..fbddfad20f 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -14,6 +14,7 @@ chrono.workspace = true chrono-tz.workspace = true croner.workspace = true flate2.workspace = true +fs4.workspace = true futures-util.workspace = true getrandom.workspace = true openidconnect.workspace = true @@ -36,6 +37,7 @@ uuid.workspace = true axum-server.workspace = true proptest.workspace = true rcgen.workspace = true +rusqlite = { workspace = true, features = ["functions"] } tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs index ec32d511b9..5c65c595fe 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod authorize_session; pub mod bootstrap_user; pub mod change_administrator; pub(crate) mod deny_authorization; +pub mod recover_administrator; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs new file mode 100644 index 0000000000..4ed5b5ea12 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs @@ -0,0 +1,71 @@ +//! An exclusively owned offline transaction restores a person, never reopens bootstrap. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrators::grant_installation_admin; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{Change, OfflineRecovery, Parameter}; + +pub async fn recover_administrator( + OfflineRecovery(transaction): OfflineRecovery, + context: &AuditContext, + (principal_id, reason): (Uuid, &str), +) -> Result<(), ControlError> { + if reason.trim().is_empty() { + return Err(ControlError::Invalid( + "offline recovery requires a reason".into(), + )); + } + let (transaction, initialized) = query( + transaction, + "SELECT principal_id FROM sqb_admin_initialization WHERE singleton=1", + vec![], + ) + .await?; + if initialized.is_empty() { + return deny( + transaction, + context, + json!({"principal_id":principal_id}), + "administrator initialization has not occurred; use create-user", + ) + .await; + } + let (transaction, people) = query( + transaction, + "SELECT disabled_at FROM sqb_principals WHERE principal_id=$1 AND kind='person'", + vec![Parameter::Id(principal_id)], + ) + .await?; + let Some(person) = people.first() else { + return deny( + transaction, + context, + json!({"principal_id":principal_id}), + "offline recovery requires an existing person", + ) + .await; + }; + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=NULL WHERE principal_id=$1", + vec![Parameter::Id(principal_id)], + ) + .await?; + let (transaction, _) = + grant_installation_admin(transaction, context, principal_id, "offline_recovery").await?; + let (mut transaction, sessions) = query(transaction, + "UPDATE sqb_password_sessions SET revoked_at=$1 WHERE principal_id=$2 AND revoked_at IS NULL RETURNING session_id", + vec![Parameter::Time(now),Parameter::Id(principal_id)]).await?; + transaction = append_audit(transaction, context, (Uuid::now_v7(), "admin.offline_recovered"), + (json!({"principal_id":principal_id}),json!({"enabled":person["disabled_at"].is_null()}), + json!({"enabled":true,"role":"admin","reason":reason,"sessions_revoked":sessions.len()}))).await?; + transaction.changes.insert(Change::Credentials); + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs index 1c71cbb1a9..af51f604f4 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod changes; pub(crate) mod database; pub(crate) mod migration; +pub(crate) mod open_control_store; pub(crate) mod open_sqlite; pub(crate) mod recall_rows; pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_control_store.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_control_store.rs new file mode 100644 index 0000000000..db9746b2f7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_control_store.rs @@ -0,0 +1,39 @@ +//! Every normal store handle holds shared access; offline recovery obtains exclusive access. + +use std::time::Duration; + +use crate::store::_helpers::open_sqlite::{open_sqlite, sqlite_path}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn open_control_store( + url: &str, + offline: bool, +) -> Result { + if url.starts_with("postgres://") || url.starts_with("postgresql://") { + let mut options = sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .acquire_timeout(Duration::from_secs(30)); + if offline { + options = options.max_connections(1); + } else { + options = options + .min_connections(1) + .idle_timeout(None) + .max_lifetime(None) + .after_connect(|connection, _| { + Box::pin(async move { + sqlx_core::query::query( + "SELECT pg_advisory_lock_shared(1936810573, hashtext(current_schema()))", + ) + .execute(connection) + .await?; + Ok(()) + }) + }); + } + return Ok(ControlStore::Postgres(options.connect(url).await?)); + } + let path = sqlite_path(url)?; + tokio::task::spawn_blocking(move || open_sqlite(path, offline)).await? +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs index 9408ebaa14..3eb9f17512 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -1,9 +1,11 @@ //! Configure a file-backed SQLite control store without interpreting filenames as URIs. use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::Duration; use crate::store::errors::ControlError; +use crate::store::models::ControlStore; pub(crate) fn sqlite_path(url: &str) -> Result { let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); @@ -18,20 +20,54 @@ pub(crate) fn sqlite_path(url: &str) -> Result { Ok(path) } -pub(crate) fn open_sqlite(path: PathBuf) -> Result<(), ControlError> { +pub(crate) fn open_sqlite(path: PathBuf, offline: bool) -> Result { + if offline && !path.is_file() { + return Err(ControlError::Invalid( + "offline recovery requires an existing initialized control store".into(), + )); + } if let Some(parent) = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) { std::fs::create_dir_all(parent)?; } + let absolute = std::env::current_dir()?.join(path); + let path = + if absolute.exists() { + absolute.canonicalize()? + } else { + absolute + .parent() + .ok_or_else(|| ControlError::Invalid("control store requires a filename".into()))? + .canonicalize()? + .join(absolute.file_name().ok_or_else(|| { + ControlError::Invalid("control store requires a filename".into()) + })?) + }; + let mut lock_path = path.as_os_str().to_owned(); + lock_path.push(".access.lock"); + let access = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path)?; + let locked = if offline { + fs4::FileExt::try_lock(&access) + } else { + fs4::FileExt::try_lock_shared(&access) + }; + locked.map_err(|error| ControlError::Invalid(format!( + "control store is in use; offline recovery requires exclusive access with all server and administration processes stopped: {error}" + )))?; let connection = rusqlite::Connection::open_with_flags( - path, + &path, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE | rusqlite::OpenFlags::SQLITE_OPEN_CREATE | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_secs(30))?; connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; - Ok(()) + Ok(ControlStore::Sqlite(path, Arc::new(access))) } diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index 4c86b0da5a..9daa4a659b 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -15,9 +15,10 @@ pub(crate) async fn begin( mode: TransactionMode, ) -> Result { let (connection, notifications) = match store { - ControlStore::Sqlite(path) => { + ControlStore::Sqlite(path, access) => { let path = path.clone(); - let (connection, notifications) = tokio::task::spawn_blocking(move || { + let access = access.clone(); + tokio::task::spawn_blocking(move || { let connection = rusqlite::Connection::open_with_flags( &path, OpenFlags::SQLITE_OPEN_READ_WRITE @@ -35,10 +36,9 @@ pub(crate) async fn begin( } else { None }; - Ok::<_, ControlError>((connection, notifications)) + Ok::<_, ControlError>((Connection::Sqlite(connection, access), notifications)) }) - .await??; - (Connection::Sqlite(connection), notifications) + .await?? } ControlStore::Postgres(pool) => { let mut transaction = pool.begin().await?; diff --git a/crates/sqlbuild-orchestration/src/store/main/begin_offline_recovery.rs b/crates/sqlbuild-orchestration/src/store/main/begin_offline_recovery.rs new file mode 100644 index 0000000000..31445de51f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/begin_offline_recovery.rs @@ -0,0 +1,26 @@ +//! Fail closed if any supported store client still holds access to this installation. + +use crate::store::_helpers::open_control_store::open_control_store; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, query::query}; +use crate::store::models::{ControlStore, OfflineRecovery, TransactionMode}; + +pub async fn begin_offline_recovery(url: &str) -> Result { + let store = open_control_store(url, true).await?; + let transaction = begin(&store, TransactionMode::Write).await?; + if matches!(store, ControlStore::Postgres(_)) { + let (transaction, rows) = query(transaction, + "SELECT pg_try_advisory_xact_lock(1936810573, hashtext(current_schema())) AS exclusive_access", + vec![]).await?; + if rows + .first() + .is_none_or(|row| row["exclusive_access"] != true) + { + return Err(ControlError::Invalid( + "offline recovery requires exclusive control-store access; stop all server and administration processes".into(), + )); + } + return Ok(OfflineRecovery(transaction)); + } + Ok(OfflineRecovery(transaction)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs index cc72f730ed..f3a7bd32d9 100644 --- a/crates/sqlbuild-orchestration/src/store/main/commit.rs +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -11,9 +11,10 @@ pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), Contro .take() .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))? { - Connection::Sqlite(connection) => { + Connection::Sqlite(connection, access) => { tokio::task::spawn_blocking(move || { - connection.execute_batch("COMMIT")?; + let connection = (connection, access); + connection.0.execute_batch("COMMIT")?; if let Some(sender) = transaction.notifications { for message in messages { let _ = sender.send(message); diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index 9796d266c4..51a5d55c56 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod append_control_event; pub(crate) mod append_recall; pub(crate) mod begin; +pub mod begin_offline_recovery; pub(crate) mod commit; pub(crate) mod compress; pub(crate) mod decode_row; diff --git a/crates/sqlbuild-orchestration/src/store/main/open_store.rs b/crates/sqlbuild-orchestration/src/store/main/open_store.rs index db8a450f4f..36dd290b01 100644 --- a/crates/sqlbuild-orchestration/src/store/main/open_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/open_store.rs @@ -1,23 +1,10 @@ -//! Open a control database; no Python or warehouse connection is involved. +//! Open a shared-access control database without Python or warehouse connections. -use std::time::Duration; - -use crate::store::_helpers::open_sqlite::{open_sqlite, sqlite_path}; +use crate::store::_helpers::open_control_store::open_control_store; use crate::store::errors::ControlError; use crate::store::models::ControlStore; /// Open PostgreSQL by URL, or SQLite by an explicit filesystem path. pub async fn open_store(url: &str) -> Result { - if url.starts_with("postgres://") || url.starts_with("postgresql://") { - let pool = sqlx_postgres::PgPoolOptions::new() - .max_connections(10) - .acquire_timeout(Duration::from_secs(30)) - .connect(url) - .await?; - return Ok(ControlStore::Postgres(pool)); - } - let path = sqlite_path(url)?; - let configure_path = path.clone(); - tokio::task::spawn_blocking(move || open_sqlite(configure_path)).await??; - Ok(ControlStore::Sqlite(path)) + open_control_store(url, false).await } diff --git a/crates/sqlbuild-orchestration/src/store/main/query.rs b/crates/sqlbuild-orchestration/src/store/main/query.rs index 9e79cdc7a1..b338d03cbc 100644 --- a/crates/sqlbuild-orchestration/src/store/main/query.rs +++ b/crates/sqlbuild-orchestration/src/store/main/query.rs @@ -15,14 +15,15 @@ pub(crate) async fn query( .take() .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))?; let rows: Vec = match connection { - Connection::Sqlite(connection) => { + Connection::Sqlite(connection, access) => { let sql = sql.to_owned(); + let connection = (connection, access); let (connection, result) = tokio::task::spawn_blocking(move || { - let result = sqlite_query(&connection, &sql, ¶meters); + let result = sqlite_query(&connection.0, &sql, ¶meters); (connection, result) }) .await?; - transaction.connection = Some(Connection::Sqlite(connection)); + transaction.connection = Some(Connection::Sqlite(connection.0, connection.1)); result? } Connection::Postgres(mut connection) => { diff --git a/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs b/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs index 1513d649ce..6869b8a9f4 100644 --- a/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs +++ b/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs @@ -17,7 +17,7 @@ pub(crate) async fn subscribe_changes( ControlError, > { match store { - ControlStore::Sqlite(path) => { + ControlStore::Sqlite(path, _) => { let path = path.clone(); Ok(( tokio::task::spawn_blocking(move || sqlite_changes(&path)).await??, diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index 4f1ddeb631..f8bae867ce 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -2,6 +2,7 @@ use std::collections::BTreeSet; use std::path::PathBuf; +use std::sync::Arc; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -13,10 +14,14 @@ use uuid::Uuid; /// An installation's control store. Workers and locations never receive this handle. #[derive(Clone, Debug)] pub enum ControlStore { - Sqlite(PathBuf), + Sqlite(PathBuf, Arc), Postgres(sqlx_postgres::PgPool), } +/// Only the offline opener can establish the exclusive-access capability. +#[derive(Debug)] +pub struct OfflineRecovery(pub(crate) ControlTransaction); + /// The first copy wins; duplicate events retain their original storage cursor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RecallAppendOutcome { @@ -54,7 +59,7 @@ pub(crate) struct ControlTransaction { #[derive(Debug)] pub(crate) enum Connection { - Sqlite(rusqlite::Connection), + Sqlite(rusqlite::Connection, Arc), Postgres(sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>), } diff --git a/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs index 9a7ff3e5d9..5381e49cd7 100644 --- a/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs @@ -99,7 +99,7 @@ pub(crate) async fn restore_snapshot_commits(store: &ControlStore) { async fn execute_fault_sql(store: &ControlStore, (sqlite, postgres): (&str, &str)) { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs index 6d9f455567..5f468f5a71 100644 --- a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -24,7 +24,7 @@ pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputC pub(crate) async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!("SQLite store") }; let connection = rusqlite::Connection::open(path).expect("inspect SQLite bytes"); diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs index 2debfcd9b6..54719f6312 100644 --- a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -4,7 +4,7 @@ use crate::store::models::ControlStore; pub(crate) async fn execute(store: &ControlStore, sql: &str) { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!("SQLite store") }; rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index 66fa0571ae..cf1f753f11 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -4,7 +4,7 @@ pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ (SELECT COUNT(*) FROM sqb_security_audit WHERE action='location.created')"; let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; return rusqlite::Connection::open(path) @@ -20,7 +20,7 @@ pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; let connection = rusqlite::Connection::open(path).expect("inspect durable tokens"); diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index debcc4d815..23bd61658c 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -10,6 +10,8 @@ mod test_administrators; mod test_contract; #[path = "passwords/test_contract.rs"] mod test_passwords; +#[path = "test_recovery.rs"] +mod test_recovery; #[path = "sessions/test_contract.rs"] mod test_sessions; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs new file mode 100644 index 0000000000..357439362f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs @@ -0,0 +1,328 @@ +//! Recovery is exclusive, atomic, audited, and cannot reopen first-admin initialization. + +use std::time::Duration; + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{ + bootstrap_user::bootstrap_user, recover_administrator::recover_administrator, +}; +use crate::credentials::main::{ + authenticate_session::authenticate_session, create_principal::create_principal, + login_password::login_password, set_password::set_password, +}; +use crate::credentials::models::{ + AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind, SessionAccess, +}; +use crate::credentials::tests::helpers::authentication_revision; +use crate::credentials::tests::test_types::AdministratorRecoveryTestCase; +use crate::store::main::{ + begin::begin, begin_offline_recovery::begin_offline_recovery, commit::commit, + migrate_store::migrate_store, open_store::open_store, query::query, +}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::store_url; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_lost_administrator_access_when_offline_recovery_runs_then_authority_and_audit_commit_atomically() + { + let test_cases = [ + AdministratorRecoveryTestCase { + description: "SQLite exclusive audited recovery", + backend: Backend::Sqlite, + expected_recoveries: 1, + }, + AdministratorRecoveryTestCase { + description: "PostgreSQL exclusive audited recovery", + backend: Backend::Postgres, + expected_recoveries: 1, + }, + ]; + for test_case in test_cases { + run(async { + let mut fixture = open_test_store(test_case.backend).await.expect("database"); + migrate_store(&fixture.store).await.expect("migrate"); + let url = store_url(&fixture.store); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted local peer")), + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json", + ); + let inputs: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop recovery")) + .expect("inputs"); + let reason = inputs["recovery_reason"].as_str().expect("recovery reason"); + let name = inputs["first_person"].as_str().expect("person name"); + let password = inputs["password"].as_str().expect("password"); + let ordinary = create_principal( + &fixture.store, + &context, + ("customers-reader", PrincipalKind::Person), + ) + .await + .expect("ordinary person"); + close_store(fixture.store).await; + assert!( + recover_administrator( + begin_offline_recovery(&url) + .await + .expect("offline empty installation"), + &context, + (ordinary, reason) + ) + .await + .is_err(), + "recovery cannot initialize an admin" + ); + fixture.store = open_store(&url).await.expect("reopen before bootstrap"); + let principal = bootstrap_user(&fixture.store, &context, name) + .await + .expect("first admin"); + set_password( + &fixture.store, + &context, + PasswordAuthority::Operator(principal), + password, + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(&fixture.store, &context, (name, password)) + .await + .expect("login") + else { + panic!("valid password"); + }; + assert!( + begin_offline_recovery(&url).await.is_err(), + "even idle normal clients exclude recovery" + ); + let (transaction, _) = query( + begin(&fixture.store, TransactionMode::Write) + .await + .expect("simulate loss"), + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![ + Parameter::Time(chrono::Utc::now()), + Parameter::Id(principal), + ], + ) + .await + .expect("disable administrator"); + let now = transaction.now; + let (transaction, _) = query(transaction, "UPDATE sqb_role_assignments SET revoked_at=$1,active_key=NULL WHERE principal_id=$2", vec![Parameter::Time(now), Parameter::Id(principal)]).await.expect("lose administrator assignment"); + commit(transaction).await.expect("commit simulated loss"); + let revision = authentication_revision(&fixture.store) + .await + .expect("revision"); + for (table, operation) in [ + ("sqb_security_audit", "INSERT"), + ("sqb_role_assignments", "INSERT"), + ("sqb_password_sessions", "UPDATE"), + ("sqb_authentication_revision", "UPDATE"), + ] { + let fault = TransactionFault { + description: "recovery publication fails", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'recovery unavailable')", + postgres_body: "RAISE EXCEPTION 'recovery unavailable'", + }; + install_fault(&fixture.store, &fault).await; + close_store(fixture.store).await; + let recovery = begin_offline_recovery(&url) + .await + .expect("exclusive access after stop"); + assert!( + recover_administrator(recovery, &context, (principal, reason)) + .await + .is_err() + ); + fixture.store = open_store(&url) + .await + .expect("reopen after failed recovery"); + remove_fault(&fixture.store, &fault).await; + assert_eq!( + authentication_revision(&fixture.store) + .await + .expect("revision after rollback"), + revision + ); + let (transaction, rows) = query(begin(&fixture.store, TransactionMode::Read).await.expect("inspect rollback"), + "SELECT (SELECT COUNT(*) FROM sqb_principals WHERE disabled_at IS NOT NULL) AS disabled, (SELECT COUNT(*) FROM sqb_role_assignments WHERE revoked_at IS NULL) AS roles, (SELECT COUNT(*) FROM sqb_password_sessions WHERE revoked_at IS NULL) AS sessions, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.offline_recovered') AS recovered", vec![]).await.expect("rollback counts"); + assert_eq!( + serde_json::to_value(&rows[0]).expect("row"), + json!({"disabled":1,"roles":0,"sessions":1,"recovered":0}) + ); + commit(transaction).await.expect("finish rollback read"); + } + close_store(fixture.store).await; + for (id, explanation) in [(principal, " "), (Uuid::nil(), reason)] { + assert!( + recover_administrator( + begin_offline_recovery(&url) + .await + .expect("offline invalid request"), + &context, + (id, explanation) + ) + .await + .is_err() + ); + } + let recovery = begin_offline_recovery(&url) + .await + .expect("offline recovery capability"); + assert!( + !matches!( + tokio::time::timeout(Duration::from_millis(100), open_store(&url)).await, + Ok(Ok(_)) + ), + "normal clients cannot enter recovery" + ); + recover_administrator(recovery, &context, (principal, reason)) + .await + .expect("restore administrator"); + fixture.store = open_store(&url) + .await + .expect("reopen recovered installation"); + assert!( + authenticate_session(&fixture.store, &session.bearer, SessionAccess::Read) + .await + .expect("old session") + .is_none() + ); + assert!(matches!( + login_password(&fixture.store, &context, (name, password)) + .await + .expect("fresh password login"), + PasswordLogin::Authenticated(_) + )); + assert!( + bootstrap_user(&fixture.store, &context, "replacement-admin") + .await + .is_err(), + "recovery never reopens bootstrap" + ); + let (transaction, rows) = query(begin(&fixture.store, TransactionMode::Read).await.expect("inspect recovery"), + "SELECT actor, granting_role, after_json FROM sqb_security_audit WHERE action='admin.offline_recovered'", vec![]).await.expect("durable recovery audit"); + assert_eq!( + rows.len(), + test_case.expected_recoveries, + "{}", + test_case.description + ); + assert_eq!(rows[0]["actor"], "local-control-store-owner"); + assert_eq!(rows[0]["granting_role"], "control-store-owner"); + let after = &rows[0]["after_json"]; + assert_eq!(after["reason"], reason); + assert_eq!(after["sessions_revoked"], 1); + commit(transaction).await.expect("finish audit read"); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_cancelled_sqlite_query_when_blocking_work_continues_then_exclusive_access_stays_locked_until_rollback() + { + let test_cases = [AdministratorRecoveryTestCase { + description: "SQLite cancellation cannot release exclusive access early", + backend: Backend::Sqlite, + expected_recoveries: 0, + }]; + for test_case in test_cases { + run(async { + let mut fixture = open_test_store(test_case.backend).await.expect("database"); + migrate_store(&fixture.store).await.expect("migrate"); + let url = store_url(&fixture.store); + close_store(fixture.store).await; + let crate::store::models::OfflineRecovery(mut transaction) = + begin_offline_recovery(&url) + .await + .expect("exclusive transaction"); + let crate::store::models::Connection::Sqlite(connection, _) = + transaction.connection.as_mut().expect("owned connection") + else { + unreachable!() + }; + let (started, observed) = std::sync::mpsc::channel(); + let (release, wait) = std::sync::mpsc::channel(); + connection + .create_scalar_function( + "wait_for_release", + 0, + rusqlite::functions::FunctionFlags::SQLITE_UTF8, + move |_| { + started.send(()).expect("notify query start"); + wait.recv().expect("release blocking query"); + Ok(1_i64) + }, + ) + .expect("blocking database function"); + let work = tokio::spawn(query(transaction, "SELECT wait_for_release()", vec![])); + tokio::task::spawn_blocking(move || { + observed + .recv_timeout(Duration::from_secs(5)) + .expect("query started") + }) + .await + .expect("observe SQLite query"); + work.abort(); + assert!(work.await.expect_err("query task cancelled").is_cancelled()); + assert!( + open_store(&url).await.is_err(), + "normal readers cannot enter an unfinished offline transaction" + ); + assert!( + begin_offline_recovery(&url).await.is_err(), + "blocking SQLite work retains the access guard after cancellation" + ); + release.send(()).expect("finish query"); + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..500 { + let Ok(recovery) = begin_offline_recovery(&url).await else { + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + }; + drop(recovery); + return; + } + panic!("exclusive access never resumed"); + }) + .await + .expect("exclusive access after rollback"); + fixture.store = open_store(&url).await.expect("normal access resumes"); + assert!(matches!(fixture.store, ControlStore::Sqlite(_, _))); + let (transaction, rows) = query( + begin(&fixture.store, TransactionMode::Read) + .await + .expect("inspect abandoned recovery"), + "SELECT event_id FROM sqb_security_audit WHERE action='admin.offline_recovered'", + vec![], + ) + .await + .expect("no successful recovery audit"); + assert_eq!( + rows.len(), + test_case.expected_recoveries, + "{}", + test_case.description + ); + commit(transaction) + .await + .expect("finish abandoned recovery read"); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index 363d89bd7d..c9e7f3cb76 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -12,3 +12,9 @@ pub(crate) struct AdministratorTestCase { pub(crate) backend: Backend, pub(crate) expected_administrators: usize, } + +pub(crate) struct AdministratorRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_recoveries: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs index 6b2c0e2b1d..d133ea5876 100644 --- a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -12,7 +12,7 @@ use uuid::Uuid; pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { let sql = "SELECT group_key, count(*) FROM sqb_concurrency_slots GROUP BY group_key"; let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; return rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs index c69347e480..3617575225 100644 --- a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -44,7 +44,7 @@ pub(crate) async fn start_run( pub(crate) async fn age_heartbeat(store: &ControlStore, run_id: Uuid, seconds: i64) { let timestamp = Utc::now() - TimeDelta::seconds(seconds); let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 223bc36025..929015c698 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -63,7 +63,7 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { async fn trigger_audit_rows(store: &ControlStore) -> Vec<(String, String, String, String, String)> { let sql = "SELECT CAST(event_id AS TEXT), actor, granting_role, CAST(before_json AS TEXT), CAST(after_json AS TEXT) FROM sqb_security_audit WHERE action='trigger.changed'"; let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; return rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index c2d836b808..af3a779d58 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -207,7 +207,7 @@ pub(super) async fn snapshot_contract(backend: Backend) -> Option { } fn verify_sqlite_deduplication(store: &ControlStore) { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { return; }; let connection = rusqlite::Connection::open(path).expect("inspect control store"); diff --git a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs index 8c221edcc0..699304e9a7 100644 --- a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -6,7 +6,7 @@ use crate::store::tests::fault_types::TransactionFault; pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; rusqlite::Connection::open(path) @@ -27,7 +27,7 @@ pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault pub(crate) async fn remove_fault(store: &ControlStore, point: &TransactionFault) { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs index 59f067dc23..0c04e6fea1 100644 --- a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -4,7 +4,7 @@ use crate::store::models::ControlStore; pub(crate) async fn expire_lease(store: &ControlStore) { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; rusqlite::Connection::open(path) diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs index 443b0e5e26..f954da5900 100644 --- a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -2,19 +2,25 @@ use crate::store::main::open_store::open_store; use crate::store::models::ControlStore; +use sqlx_core::connection::ConnectOptions; pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + open_store(&store_url(store)) + .await + .expect("open independent control-store handle") +} + +pub(crate) fn store_url(store: &ControlStore) -> String { let ControlStore::Postgres(pool) = store else { - let ControlStore::Sqlite(path) = store else { + let ControlStore::Sqlite(path, _) = store else { unreachable!() }; - return open_store(path.to_str().expect("UTF-8 test path")) - .await - .expect("open independent SQLite handle"); + return path.to_str().expect("UTF-8 test path").to_owned(); }; - ControlStore::Postgres( - sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) - .await - .expect("open independent PostgreSQL pool"), - ) + let options = pool.connect_options(); + let mut url = options.to_url_lossy(); + // SQLx's lossy URL omits startup options, including the isolated schema search path. + url.query_pairs_mut() + .extend_pairs(options.get_options().map(|startup| ("options", startup))); + url.to_string() } diff --git a/deploy/README.md b/deploy/README.md index a4bef43bfd..b8421b7007 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -127,7 +127,17 @@ use `POST /api/admin/people/{principal_id}` with a JSON action of `grant`, `revo or `enable`, their session cookie and CSRF header. This cannot remove or disable the final enabled, password-provisioned administrator; an unprovisioned person is not a usable alternative in this password-only administration path. It never authorizes a machine bearer or grants non-admin defaults. -The explicit offline recovery command is not yet available. + +If administrator access is lost, stop every server and administration process connected to the +installation, then run `sqb admin recover-user PRINCIPAL_ID --reason "restore administrator access"` +as its local control-store owner. Recovery requires exclusive store access, enables an existing +person, restores installation admin membership, revokes that person's browser sessions, and records +the reason in the same durable transaction. It never reopens first-user initialization or creates a +person. Provision a replacement password with `set-password` if needed, restart the services and sign +in again. SQLite clients coordinate through an adjacent `.access.lock` file: retain that file and +use the same database path for all clients. PostgreSQL clients coordinate through advisory locks in +the installation schema. These locks protect supported SQLBuild clients, not arbitrary database-owner +SQL or older clients that do not implement this protocol; those processes must also be stopped. An installation administrator assigns or revokes additive built-in roles through `POST /api/admin/roles`, using `principal_id`, `role`, optional `location_id` and `target`, From a5ebdc89b65d0c02f865b05a68cc5bb6f1cd9f22 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:49:04 +0000 Subject: [PATCH 080/138] feat: configure optional viewer metadata access --- .../sqlbuild-cli/src/argv/command_table.json | 9 +++ .../argv/tests/test_parse_cli_arguments.rs | 12 ++++ .../src/native_commands/_helpers/decision.rs | 4 ++ .../api/_helpers/authentication/configure.rs | 6 +- .../_helpers/resources/person_operations.rs | 4 +- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../authorization/main/assign_builtin_role.rs | 5 +- .../authorization/main/authorize_session.rs | 7 ++- .../main/change_administrator.rs | 5 +- .../src/credentials/models.rs | 9 +++ .../main/change_person_trigger.rs | 11 +++- .../src/runs/_helpers/requests/manual_job.rs | 20 +++++-- .../src/runs/main/list_person_runs.rs | 4 +- .../tests/api/test_passwords.rs | 52 ++++++++++++++++- .../tests/runs/test_permissions.rs | 5 +- deploy/README.md | 9 ++- .../publication/_test_types.py | 8 +++ .../publication/test_person_journey.py | 58 ++++++++++++++----- 18 files changed, 193 insertions(+), 36 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f781b02f9d..c17fbc2ced 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1081,6 +1081,15 @@ "help": "enable password sessions for this exact external HTTPS origin; requires HTTPS termination in front of the HTTP listener", "group": 1 }, + { + "kind": "store", + "options": ["--default-role"], + "dest": "control_default_role", + "choices": ["none", "viewer"], + "default": "none", + "help": "ambient metadata access for authenticated people: none (default) or installation-wide viewer; never grants SQL, evidence, execution or administration", + "group": 1 + }, { "kind": "store", "options": ["--trusted-proxy-config"], diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index 62ba60f4db..2524c91185 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -49,6 +49,18 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "contract_command", expected_value: Value::Str("diff".to_owned()), }, + ParsedValueTestCase { + description: "server grants no ambient role by default", + argv: &["server"], + attribute: "control_default_role", + expected_value: Value::Str("none".into()), + }, + ParsedValueTestCase { + description: "viewer metadata access requires an explicit server option", + argv: &["server", "--default-role", "viewer"], + attribute: "control_default_role", + expected_value: Value::Str("viewer".into()), + }, ParsedValueTestCase { description: "an explicit failed-node rerun never becomes a full rerun", argv: &[ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 0681d30b12..a86f0cca60 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -265,6 +265,10 @@ pub(crate) fn control_authentication( ) -> Result { Ok(ApiAuthentication { password_origin: text(namespace, "control_password_origin"), + default_role: serde_json::from_value(json!( + text(namespace, "control_default_role").unwrap_or_else(|| "none".into()) + )) + .map_err(|_| ControlError::Invalid("default role must be none or viewer".into()))?, trusted_proxy: authentication_settings( text(namespace, "control_trusted_proxy_config"), ("trusted-proxy", "origin, trusted_peers and identities"), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs index 7619f94542..a39792d05e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; -use axum::Router; +use axum::{Extension, Router}; use crate::api::_helpers::authentication::{oidc, password_sessions, trusted_proxy}; use crate::api::models::{ApiAuthentication, DevelopmentScope}; @@ -19,6 +19,7 @@ pub(crate) async fn configure( password_origin, trusted_proxy, oidc, + default_role, } = authentication; if development_scope.is_some() && password_origin.is_some() { return Err(ControlError::Invalid( @@ -46,6 +47,7 @@ pub(crate) async fn configure( development_scope, password_sessions::router(store, password_origin)? .merge(trusted_proxy::router(store, trusted_proxy)?) - .merge(oidc::router(store, oidc).await?), + .merge(oidc::router(store, oidc).await?) + .layer(Extension(default_role)), )) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index bc6296fef1..0db9840399 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -8,7 +8,7 @@ use uuid::Uuid; use crate::api::_helpers::authentication::password_transport::session_credentials; use crate::api::errors::ApiError; use crate::api::models::{MutationResponse, ReadQuery}; -use crate::credentials::models::AuditContext; +use crate::credentials::models::{AuditContext, DefaultRole}; use crate::declarations::main::change_person_trigger::change_person_trigger; use crate::declarations::models::TriggerChangeRequest; use crate::runs::main::list_person_runs::list_person_runs; @@ -56,6 +56,7 @@ pub(crate) async fn change_trigger( (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] pub(crate) async fn recent_runs( (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), headers: HeaderMap, ) -> Result>, ApiError> { @@ -64,6 +65,7 @@ pub(crate) async fn recent_runs( &store, (&context, session_credentials(&headers)), (location_id, &target, options.limit.unwrap_or(100)), + default_role, ) .await?, )) diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 4d6984d45f..9049cff92e 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -112,6 +112,7 @@ pub struct ApiAuthentication { pub password_origin: Option, pub trusted_proxy: Option, pub oidc: Option, + pub default_role: crate::credentials::models::DefaultRole, } /// Only the server's installation configuration selects the daemon and container to inspect. diff --git a/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs index 43622bf833..7f2c3bb62c 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs @@ -2,7 +2,9 @@ use crate::authorization::_helpers::administrator_roles::assign_role_in; use crate::authorization::main::authorize_session::authorize_session; -use crate::credentials::models::{Action, AuditContext, PermissionScope, RoleAssignmentRequest}; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, PermissionScope, RoleAssignmentRequest, +}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::models::{ControlStore, TransactionMode}; @@ -21,6 +23,7 @@ pub async fn assign_builtin_role( target: None, nodes: Default::default(), }, + DefaultRole::None, ) .await?; assign_role_in(transaction, &context, request).await diff --git a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs index 8ee18a3cff..e01232df14 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs @@ -5,7 +5,7 @@ use serde_json::json; use crate::authorization::main::deny_authorization::deny; use crate::credentials::main::authenticate_session_in::authenticate_session_in; use crate::credentials::models::{ - Action, AuditContext, BuiltinRole, PermissionScope, SessionAccess, + Action, AuditContext, BuiltinRole, DefaultRole, PermissionScope, SessionAccess, }; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -40,6 +40,7 @@ pub(crate) async fn authorize_session( transaction: ControlTransaction, (context, credentials): (&AuditContext, (&str, &str)), scope: &PermissionScope, + default_role: DefaultRole, ) -> Result<(ControlTransaction, AuditContext), ControlError> { let access = match scope.action { Action::View | Action::ViewSql | Action::ViewEvidence | Action::AuditView => { @@ -83,6 +84,10 @@ pub(crate) async fn authorize_session( return Ok((transaction, context)); } } + if default_role == DefaultRole::Viewer && role_allows(BuiltinRole::Viewer, scope.action) { + context.granting_role = "default:viewer".into(); + return Ok((transaction, context)); + } deny( transaction, &context, diff --git a/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs index ffd44f1494..d07be01fbe 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs @@ -4,7 +4,9 @@ use uuid::Uuid; use crate::authorization::_helpers::administrator_changes::apply_change; use crate::authorization::main::authorize_session::authorize_session; -use crate::credentials::models::{Action, AdministratorChange, AuditContext, PermissionScope}; +use crate::credentials::models::{ + Action, AdministratorChange, AuditContext, DefaultRole, PermissionScope, +}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::models::{ControlStore, TransactionMode}; @@ -27,6 +29,7 @@ pub async fn change_administrator( target: None, nodes: Default::default(), }, + DefaultRole::None, ) .await?; apply_change(transaction, &context, (principal_id, change)).await diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index e447c8bfcf..b91d6f3945 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -88,6 +88,15 @@ pub enum BuiltinRole { Admin, } +/// Installation-owned ambient access; never permits an administrative or execution default. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DefaultRole { + #[default] + None, + Viewer, +} + /// Scope is computed by the server from the operation, never accepted as a claimed write set. #[derive(Clone, Debug, Serialize)] pub(crate) struct PermissionScope { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs index e44ae32a92..c9e0a2948e 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs @@ -4,7 +4,7 @@ use serde_json::json; use crate::authorization::main::authorize_session::authorize_session; use crate::authorization::main::deny_authorization::deny; -use crate::credentials::models::{Action, AuditContext, PermissionScope}; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::declarations::_helpers::operator_change::change_trigger_in; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; use crate::store::errors::ControlError; @@ -37,8 +37,13 @@ pub async fn change_person_trigger( .map_err(|error| ControlError::Invalid(error.to_string()))?, nodes: Default::default(), }; - let (transaction, context) = - authorize_session(transaction, (context, credentials), &scope).await?; + let (transaction, context) = authorize_session( + transaction, + (context, credentials), + &scope, + DefaultRole::None, + ) + .await?; if request.reason.trim().is_empty() { return deny( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index acbff53245..b9da47bd8f 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::authorization::main::authorize_session::authorize_session; use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{Action, AuditContext, PermissionScope}; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::job::frozen_group_children; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; use crate::runs::main::frozen_job_request::frozen_job_request; @@ -26,8 +26,13 @@ pub(crate) async fn request_job_run_in( nodes: Default::default(), }; if let Some(credentials) = credentials { - (transaction, context) = - authorize_session(transaction, (&context, credentials), &scope).await?; + (transaction, context) = authorize_session( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; } if context.actor.trim().is_empty() || context.granting_role.trim().is_empty() @@ -82,8 +87,13 @@ pub(crate) async fn request_job_run_in( }) { scope.action = Action::RunRewriteHistory; - (transaction, context) = - authorize_session(transaction, (&context, credentials), &scope).await?; + (transaction, context) = authorize_session( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; } } let (transaction, _) = publish_request(transaction, &frozen).await?; diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs index b2e33af357..9d21d68a25 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs @@ -3,7 +3,7 @@ use uuid::Uuid; use crate::authorization::main::authorize_session::authorize_session; -use crate::credentials::models::{Action, AuditContext, PermissionScope}; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::lifecycle::reads::list_runs_in; use crate::runs::models::RunRecord; use crate::store::errors::ControlError; @@ -14,6 +14,7 @@ pub async fn list_person_runs( store: &ControlStore, (context, credentials): (&AuditContext, (&str, &str)), (location_id, target, limit): (Uuid, &str, u32), + default_role: DefaultRole, ) -> Result, ControlError> { let (transaction, _) = authorize_session( begin(store, TransactionMode::Write).await?, @@ -24,6 +25,7 @@ pub async fn list_person_runs( target: Some(target.into()), nodes: Default::default(), }, + default_role, ) .await?; let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; diff --git a/crates/sqlbuild-orchestration/tests/api/test_passwords.rs b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs index b961f68ade..63fc9fd73d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_passwords.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs @@ -9,7 +9,7 @@ use crate::api::models::{ApiAuthentication, AuthenticationDenial, PasswordSessio use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::PasswordApiTestCase; use crate::credentials::main::{create_principal::create_principal, set_password::set_password}; -use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; +use crate::credentials::models::{AuditContext, DefaultRole, PasswordAuthority, PrincipalKind}; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, }; @@ -91,6 +91,7 @@ fn given_password_http_sessions_when_changed_or_revoked_then_cookie_and_csrf_bou &replica, ApiAuthentication { password_origin: Some(origin.into()), + default_role: DefaultRole::Viewer, ..ApiAuthentication::default() }, ) @@ -221,6 +222,55 @@ fn given_password_http_sessions_when_changed_or_revoked_then_cookie_and_csrf_bou .await .expect("second replica read"); assert_eq!(response.status(), StatusCode::OK); + let location_id = Uuid::now_v7(); + for (server, expected_status) in + [(&api, StatusCode::FORBIDDEN), (&other, StatusCode::OK)] + { + let url = format!( + "{}/api/control/locations/{location_id}/targets/development/runs", + server.url + ); + assert_eq!( + client + .get(&url) + .send() + .await + .expect("unauthenticated read") + .status(), + StatusCode::UNAUTHORIZED, + "viewer default cannot replace authentication" + ); + let response = client + .get(url) + .header(COOKIE, &cookie) + .send() + .await + .expect("live server-owned default role"); + assert_eq!(response.status(), expected_status); + assert_eq!( + response + .json::() + .await + .expect("metadata response") + .as_array() + .map(Vec::len), + (expected_status == StatusCode::OK).then_some(0), + "only the configured viewer receives the empty metadata array" + ); + let response = client + .post(format!("{}/api/control/runs", server.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", &issued.csrf_token) + .json( + &json!({"run_id": Uuid::now_v7(), "location_id": location_id, + "target": "development", "job_key": "daily_orders", + "reason": "Build orders", "priority": 0}), + ) + .send() + .await + .expect("default role cannot execute"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } for csrf in cases["invalid_csrf"].as_array().expect("invalid CSRF") { let response = client .post(format!("{}/api/auth/logout", other.url)) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index d1f32fb678..1c5c1a8681 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -11,7 +11,7 @@ use crate::credentials::main::{ create_principal::create_principal, login_password::login_password, set_password::set_password, }; use crate::credentials::models::{ - AdministratorChange, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, + AdministratorChange, AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, PasswordLogin, PrincipalKind, RoleAssignmentRequest, }; use crate::credentials::tests::location_helpers::create_location; @@ -166,7 +166,8 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi list_person_runs( store, (&context, credentials), - (location_id, "development", 10) + (location_id, "development", 10), + DefaultRole::None, ) .await .expect("built-in metadata access") diff --git a/deploy/README.md b/deploy/README.md index b8421b7007..5edd137adf 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -150,8 +150,13 @@ for recent run metadata, `POST /api/control/runs` for published-job requests, an `POST /api/control/triggers` for trigger overrides. Viewers can read metadata but cannot run jobs; operators and developers can run ordinary jobs and toggle triggers. Full-refresh jobs, published cursor bounds, explicit backfill windows and cursor resets require an installation -administrator in these endpoints. A person with no assigned role cannot use them. Metadata responses do not -contain SQL, sampled rows or captured output. These person routes do not accept machine +administrator in these endpoints. The server defaults to `--default-role none`: a person with no +assigned role cannot use them. Set `--default-role viewer` to give every authenticated person +installation-wide metadata access, including people whose explicit roles have been revoked. +This is a live server setting, not a persisted assignment; returning it to `none` removes that +ambient access without changing explicit grants. It never permits SQL, evidence, job execution +or administration. Metadata responses do not contain SQL, sampled rows or captured output. +These person routes do not accept machine credentials; location and worker credentials continue to use the separate machine routes. Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in `.env` before creating the server to receive the native diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 1c7fae7b54..0b58b14ab8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,6 +1,7 @@ """Publication cases and isolated server credentials.""" from collections.abc import Callable +from contextlib import AbstractContextManager from dataclasses import dataclass, field from datetime import datetime from pathlib import Path @@ -109,6 +110,13 @@ class WorkerExecutionTestCase: launcher: Literal["native", "uncooperative", "docker"] = "native" +@dataclass(frozen=True, kw_only=True) +class PersonJourneyTestCase(WorkerExecutionTestCase): + description: str + default_role: Literal["none", "viewer"] + expected_metadata_access: Callable[[], AbstractContextManager[Any]] + + @dataclass(frozen=True) class GroupExecutionTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index ce3aabb453..17a425ef1d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -2,6 +2,8 @@ import json import sys +from contextlib import nullcontext +from functools import partial from pathlib import Path from typing import Any from urllib.error import HTTPError @@ -11,7 +13,7 @@ import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( - WorkerExecutionTestCase, + PersonJourneyTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( current_snapshot, @@ -33,24 +35,42 @@ @pytest.mark.parametrize( "test_case", [ - WorkerExecutionTestCase( - "owner bootstrap, least-privilege browser operations, reload and last-good recovery", - "SELECT 23 AS amount", - "SELECT amount FROM orders", - "success", - [(23,)], - ) + PersonJourneyTestCase( + description="no-role default, scoped browser operations, reload and last-good recovery", + model_sql="SELECT 23 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + default_role="none", + expected_metadata_access=partial(pytest.raises, HTTPError, match="HTTP Error 403"), + ), + PersonJourneyTestCase( + description="configured viewer default, scoped execution, reload and last-good recovery", + model_sql="SELECT 23 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + default_role="viewer", + expected_metadata_access=nullcontext, + ), ], ids=lambda case: case.description, ) def test_given_an_initialized_installation_when_people_run_and_reload_then_permissions_and_recovery_hold( - test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: PersonJourneyTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: with scheduled_worker( tmp_path, monkeypatch, test_case, - server_arguments=("--password-origin", "https://control.example.com"), + server_arguments=( + "--password-origin", + "https://control.example.com", + "--default-role", + test_case.default_role, + ), ) as worker: scheduled: dict[str, Any] = worker_result(worker, test_case) admin: dict[str, Any] = provision_person( @@ -76,9 +96,11 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi "location_id": worker.api.location_id, "target": "development", } - with pytest.raises(HTTPError) as no_role: - person_read(worker.api, runs_path, session=session) - assert no_role.value.code == 403 + with test_case.expected_metadata_access(): + assert ( + person_read(worker.api, runs_path, session=session)[0]["run_id"] + == scheduled["run_id"] + ) for path, body in [ ("/api/control/runs", request), ("/api/admin/roles", assignment), @@ -100,9 +122,13 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi with pytest.raises(HTTPError) as rejected: person_request(worker.api, path, body=body, session=session) assert rejected.value.code == 403 - with pytest.raises(HTTPError) as wrong_target: - person_read(worker.api, runs_path.replace("development", "preview"), session=session) - assert wrong_target.value.code == 403 + with test_case.expected_metadata_access(): + assert ( + person_read( + worker.api, runs_path.replace("development", "preview"), session=session + ) + == [] + ) assignment["role"] = "operator" assert person_request( worker.api, "/api/admin/roles", body=assignment, session=admin_session From 6d0c3461ba8e678a9252a130841491a3bf78054d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 03:36:06 +0000 Subject: [PATCH 081/138] feat: authorize person cancellation and frozen reruns --- .github/workflows/verify.yml | 10 +- .../administrator_lifecycle.json | 1 + clients/typescript/src/schema.d.ts | 131 +++++++++++++ .../src/native_commands/main/_run_admin.rs | 2 +- .../authentication/password_sessions.rs | 8 + .../_helpers/resources/person_operations.rs | 38 +++- .../sqlbuild-orchestration/src/api/models.rs | 2 + .../src/runs/_helpers/lifecycle/change_run.rs | 83 ++++++++ .../src/runs/_helpers/lifecycle/mod.rs | 1 + .../src/runs/_helpers/requests/manual_job.rs | 43 +---- .../runs/_helpers/requests/manual_rerun.rs | 85 +++++++++ .../src/runs/_helpers/requests/mod.rs | 2 + .../src/runs/_helpers/requests/permissions.rs | 70 +++++++ .../src/runs/_helpers/requests/rerun.rs | 60 ++++-- .../src/runs/main/cancel_person_run.rs | 60 ++++++ .../src/runs/main/change_run.rs | 86 +-------- .../src/runs/main/frozen_job_request.rs | 2 +- .../src/runs/main/mod.rs | 1 + .../src/runs/main/request_rerun.rs | 46 +---- .../sqlbuild-orchestration/src/runs/models.rs | 11 +- .../tests/runs/test_permissions.rs | 177 +++++++++++++++++- .../tests/runs/test_reruns.rs | 30 ++- .../tests/sensors/test_runs.rs | 4 +- deploy/README.md | 16 +- .../publication/test_person_journey.py | 174 ++++++++++++++++- 25 files changed, 946 insertions(+), 197 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3b35960b21..9df9458535 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -471,7 +471,15 @@ jobs: shared-key: verify-windows save-if: false - name: Test the native discovery crate (walk, junctions, ordering, read errors) - run: cargo test --locked -p sqlbuild-discovery + run: | + status=0 + cargo test --locked -p sqlbuild-discovery || status=$? + if [ "${status}" -ne 0 ]; then + echo "Windows dependency resolution diagnostics (original failure is retained):" + cargo tree --offline -p sqlbuild-discovery || true + git diff -- Cargo.lock + fi + exit "${status}" - name: Install Python run: uv python install 3.12 - name: Install SQLBuild and pytest diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json index db7faa5392..f66bd786c5 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -4,6 +4,7 @@ "password": "orders-bootstrap-password-2026", "recovery_reason": "restore access after loss of the final administrator", "disabled_person_sequence": ["assign", "disable", "revoke", "enable", "deny_run"], + "run_control_sequence": ["request", "cancel", "confirm_stopped", "rerun_frozen", "retry", "revoke", "deny_retry"], "job_settings": [ {"full_refresh": false}, {"full_refresh": true}, diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 42d8082962..dfa038206e 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -179,6 +179,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/runs/cancel": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["cancelPersonRun"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/runs/rerun": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["rerunPersonRun"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/triggers": { parameters: { query?: never; @@ -534,6 +566,14 @@ export interface components { }; /** @enum {string} */ BuiltinRole: "viewer" | "operator" | "developer" | "admin"; + /** @description Browser cancellation cannot supply an actor, claim or lifecycle transition. */ + CancelRunRequest: { + /** Format: uuid */ + event_id: string; + reason: string; + /** Format: uuid */ + run_id: string; + }; CatchUp: "latest" | { /** @description Process the oldest bounded batch; later due times remain due for the next transaction. */ all: { @@ -820,6 +860,17 @@ export interface components { kind: "blob"; value: number[]; }; + /** @description A retry derives its execution scope from a terminal run, never from caller selectors. */ + RerunRequest: { + from_failure: boolean; + /** Format: int64 */ + priority: number; + reason: string; + /** Format: uuid */ + run_id: string; + /** Format: uuid */ + source_run_id: string; + }; RoleAssignmentRequest: { /** Format: uuid */ location_id?: string | null; @@ -1617,6 +1668,86 @@ export interface operations { }; }; }; + cancelPersonRun: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CancelRunRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + rerunPersonRun: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RerunRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; changeTrigger: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 8487fef74a..0aee19f181 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -75,7 +75,7 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { - let applied = request_rerun(&store, &context, &request).await?; + let applied = request_rerun(&store, (&context, None), &request).await?; Ok(json!({"run_id": request.run_id, "applied": applied})) } ControlAdmin::RequestLocationWork(request) => { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 5256220d30..cd15d8496b 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -49,6 +49,14 @@ pub(crate) fn router( "/api/control/runs", post(crate::api::_helpers::resources::person_operations::request_job), ) + .route( + "/api/control/runs/cancel", + post(crate::api::_helpers::resources::person_operations::cancel_run), + ) + .route( + "/api/control/runs/rerun", + post(crate::api::_helpers::resources::person_operations::rerun), + ) .route( "/api/control/triggers", post(crate::api::_helpers::resources::person_operations::change_trigger), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 0db9840399..43a89c8173 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -11,9 +11,11 @@ use crate::api::models::{MutationResponse, ReadQuery}; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::declarations::main::change_person_trigger::change_person_trigger; use crate::declarations::models::TriggerChangeRequest; +use crate::runs::main::cancel_person_run::cancel_person_run; use crate::runs::main::list_person_runs::list_person_runs; use crate::runs::main::request_person_job_run::request_person_job_run; -use crate::runs::models::{JobRunRequest, RunRecord}; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord}; use crate::store::models::ControlStore; #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", @@ -35,6 +37,40 @@ pub(crate) async fn request_job( })) } +#[utoipa::path(post, path = "/api/control/runs/cancel", operation_id = "cancelPersonRun", + params(("x-sqb-csrf" = String, Header)), request_body = CancelRunRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn cancel_run( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: cancel_person_run(&store, (&context, session_credentials(&headers)), &request) + .await?, + })) +} + +#[utoipa::path(post, path = "/api/control/runs/rerun", operation_id = "rerunPersonRun", + params(("x-sqb-csrf" = String, Header)), request_body = RerunRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn rerun( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_rerun( + &store, + (&context, Some(session_credentials(&headers))), + &request, + ) + .await?, + })) +} + #[utoipa::path(post, path = "/api/control/triggers", operation_id = "changeTrigger", params(("x-sqb-csrf" = String, Header)), request_body = TriggerChangeRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 9049cff92e..c532994002 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -15,6 +15,8 @@ use crate::api::_helpers::resources::{ administrators::change, administrators::assign, person_operations::request_job, + person_operations::cancel_run, + person_operations::rerun, person_operations::change_trigger, person_operations::recent_runs, password_sessions::login, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs new file mode 100644 index 0000000000..bc2e969b69 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs @@ -0,0 +1,83 @@ +//! Retry-safe facts, lifecycle projection and terminal claim release form one transaction. + +use serde_json::json; + +use crate::credentials::main::append_audit::append_audit; +use crate::runs::_helpers::lifecycle::changes::{ + check_launch_claim, validate_authority, validate_change, validate_retry, +}; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Use authenticated authority and record unclaimed cancellation as terminal in the same transaction. +pub(crate) async fn change_run_in( + mut transaction: ControlTransaction, + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + validate_authority(authority, request)?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + let body = json!({"change": request}); + let (transaction, previous) = query( + transaction, + "SELECT invocation_id, payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(request.event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + validate_retry(previous, request, &body)?; + return Ok((transaction, false)); + } + let (mut transaction, rows) = query( + transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, + )?; + if matches!(request.change, RunChange::LaunchFailed { .. }) { + transaction = check_launch_claim(transaction, request).await?; + } else { + validate_change(&state, request)?; + } + let mut transaction = publish_change(transaction, request, &state).await?; + if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = + (authority, &request.change) + { + let mut rows; + (transaction, rows) = query( + transaction, + "SELECT location_id, target, status, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let after = rows.remove(0); + transaction = append_audit( + transaction, + context, + (request.event_id, "run.cancel_requested"), + ( + json!({"location_id": after["location_id"], "target": after["target"], "run_id": request.run_id}), + json!({"status": state.status, "cancel_requested_at": state.cancel_requested_at}), + json!({"status": after["status"], "cancel_requested_at": after["cancel_requested_at"], "reason": reason}), + ), + ) + .await?; + } + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 6f70d90d15..b63e6414ba 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -1,3 +1,4 @@ +pub(in crate::runs) mod change_run; pub(in crate::runs) mod changes; pub(in crate::runs) mod claims; pub(in crate::runs) mod events; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index b9da47bd8f..33875643e1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -6,10 +6,10 @@ use uuid::Uuid; use crate::authorization::main::authorize_session::authorize_session; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; -use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; use crate::runs::main::frozen_job_request::frozen_job_request; -use crate::runs::models::{JobRunRequest, RunKind, RunWindow}; +use crate::runs::models::{JobRunRequest, RunWindow}; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; @@ -19,7 +19,7 @@ pub(crate) async fn request_job_run_in( request: &JobRunRequest, ) -> Result<(ControlTransaction, bool), ControlError> { let mut context = context.clone(); - let mut scope = PermissionScope { + let scope = PermissionScope { action: Action::JobRun, location_id: Some(request.location_id), target: Some(request.target.clone()), @@ -64,37 +64,12 @@ pub(crate) async fn request_job_run_in( } let (mut transaction, frozen) = frozen_job_request(transaction, &context, request).await?; if let Some(credentials) = credentials { - let mut executions = vec![frozen.clone()]; - if frozen.kind == RunKind::Group { - (transaction, executions) = frozen_group_children(transaction, &frozen).await?; - } - scope.nodes = executions - .iter() - .flat_map(|run| run.write_set.iter().cloned()) - .collect(); - if request.window.is_some() - || executions.iter().any(|run| { - run.settings - .get("full_refresh") - .and_then(serde_json::Value::as_bool) - == Some(true) - || run.settings.contains_key("start_cursor") - || run.settings.contains_key("end_cursor") - || run.settings.contains_key("start_cursor_ts") - || run.settings.contains_key("end_cursor_ts") - || run.settings.contains_key("start_cursor_int") - || run.settings.contains_key("end_cursor_int") - }) - { - scope.action = Action::RunRewriteHistory; - (transaction, context) = authorize_session( - transaction, - (&context, credentials), - &scope, - DefaultRole::None, - ) - .await?; - } + (transaction, context) = authorize_execution( + transaction, + (&context, credentials), + (&frozen, Action::JobRun), + ) + .await?; } let (transaction, _) = publish_request(transaction, &frozen).await?; let transaction = append_audit(transaction, &context, (event_id, "job.run_requested"), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs new file mode 100644 index 0000000000..2c98f2f9b2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs @@ -0,0 +1,85 @@ +//! Publish a new pinned execution and its operator audit atomically, including retry identity. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext}; +use crate::runs::_helpers::requests::permissions::authorize_execution; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::models::{RerunRequest, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// No credentials means a trusted direct-store owner, never an HTTP-supplied role. +pub(crate) async fn request_rerun_in( + mut transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option<(&str, &str)>), + request: &RerunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let mut context = context.clone(); + let mut prepared = None; + if let Some(credentials) = credentials { + let rows; + (transaction, rows) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let frozen: RunRequest; + if let Some(row) = rows.into_iter().next() { + frozen = serde_json::from_value(row["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, context) = authorize_execution( + transaction, + (&context, credentials), + (&frozen, Action::RunRerun), + ) + .await?; + } else { + (transaction, (context, frozen)) = + frozen_rerun(transaction, (&context, Some(credentials)), request).await?; + } + prepared = Some(frozen); + } + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "rerun needs an actor, granting role and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"rerun-request"); + let (transaction, retry) = request_retry( + transaction, + &context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { + return Ok((transaction, false)); + } + let (transaction, frozen) = if let Some(frozen) = prepared { + (transaction, frozen) + } else { + let (transaction, (_, frozen)) = + frozen_rerun(transaction, (&context, None), request).await?; + (transaction, frozen) + }; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + &context, + (event_id, "run.rerun_requested"), + ( + json!({"location_id": frozen.location_id, "target": frozen.target, "job_key": frozen.job_key, "run_id": request.run_id}), + json!({"source_run_id": request.source_run_id}), + json!({"request": request, "frozen": frozen}), + ), + ).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs index 0c44cb5dcd..fbd365703f 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -1,4 +1,6 @@ pub(in crate::runs) mod job; pub(in crate::runs) mod manual_job; +pub(in crate::runs) mod manual_rerun; +pub(in crate::runs) mod permissions; pub(in crate::runs) mod publication; pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs new file mode 100644 index 0000000000..9a8b9632c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -0,0 +1,70 @@ +//! Person authority covers the effective frozen execution, including sensitive group children. + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::models::{RunKind, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn authorize_execution( + mut transaction: ControlTransaction, + (context, credentials): (&AuditContext, (&str, &str)), + (request, action): (&RunRequest, Action), +) -> Result<(ControlTransaction, AuditContext), ControlError> { + let mut executions = vec![request.clone()]; + if request.kind == RunKind::Group { + (transaction, executions) = frozen_group_children(transaction, request).await?; + } + let mut scope = PermissionScope { + action, + location_id: Some(request.location_id), + target: Some(request.target.clone()), + nodes: executions + .iter() + .flat_map(|run| run.write_set.iter().cloned()) + .collect(), + }; + let (mut transaction, mut context) = authorize_session( + transaction, + (context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + if action == Action::RunRerun && request.kind == RunKind::Adhoc { + scope.action = Action::RunAdhoc; + (transaction, context) = authorize_session( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + } + if matches!(action, Action::JobRun | Action::RunRerun) + && (request.window.is_some() + || executions.iter().any(|run| { + run.settings + .get("full_refresh") + .and_then(serde_json::Value::as_bool) + == Some(true) + || run.settings.contains_key("start_cursor") + || run.settings.contains_key("end_cursor") + || run.settings.contains_key("start_cursor_ts") + || run.settings.contains_key("end_cursor_ts") + || run.settings.contains_key("start_cursor_int") + || run.settings.contains_key("end_cursor_int") + })) + { + scope.action = Action::RunRewriteHistory; + (transaction, context) = authorize_session( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + } + Ok((transaction, context)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs index cd1e9a7ecc..1dba2dc64f 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -4,7 +4,9 @@ use std::collections::BTreeSet; use serde_json::from_value; -use crate::credentials::models::AuditContext; +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -12,25 +14,34 @@ use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn frozen_rerun( transaction: ControlTransaction, - context: &AuditContext, + (context, credentials): (&AuditContext, Option<(&str, &str)>), request: &RerunRequest, -) -> Result<(ControlTransaction, RunRequest), ControlError> { +) -> Result<(ControlTransaction, (AuditContext, RunRequest)), ControlError> { let (mut transaction, rows) = query( transaction, "SELECT request_json, status FROM sqb_runs WHERE run_id=$1", vec![Parameter::Id(request.source_run_id)], ) .await?; - let source = rows - .first() - .ok_or_else(|| ControlError::Invalid("source run was not found".into()))?; + let Some(source) = rows.first() else { + if let Some(credentials) = credentials { + authorize_session( + transaction, + (context, credentials), + &PermissionScope { + action: Action::RunRerun, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + } + return Err(ControlError::Invalid("source run was not found".into())); + }; let status: RunStatus = from_value(source["status"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; - if !status.is_terminal() { - return Err(ControlError::Invalid( - "rerun requires a terminal source run".into(), - )); - } let mut frozen: RunRequest = from_value(source["request_json"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; if request.from_failure { @@ -49,11 +60,6 @@ pub(crate) async fn frozen_rerun( frozen .write_set .retain(|key| frozen.selection.contains(key)); - if frozen.selection.is_empty() { - return Err(ControlError::Invalid( - "source run has no failed or skipped nodes".into(), - )); - } } frozen.run_id = request.run_id; frozen.origin = RunOrigin::Rerun; @@ -64,5 +70,25 @@ pub(crate) async fn frozen_rerun( frozen.rerun_of_run_id = Some(request.source_run_id); frozen.requested_by = context.actor.clone(); frozen.priority = request.priority; - Ok((transaction, frozen)) + let mut context = context.clone(); + if let Some(credentials) = credentials { + (transaction, context) = authorize_execution( + transaction, + (&context, credentials), + (&frozen, Action::RunRerun), + ) + .await?; + frozen.requested_by.clone_from(&context.actor); + } + if !status.is_terminal() { + return Err(ControlError::Invalid( + "rerun requires a terminal source run".into(), + )); + } + if request.from_failure && frozen.selection.is_empty() { + return Err(ControlError::Invalid( + "source run has no failed or skipped nodes".into(), + )); + } + Ok((transaction, (context, frozen))) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs b/crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs new file mode 100644 index 0000000000..2764889adf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs @@ -0,0 +1,60 @@ +//! Cancellation derives scope and actor from durable execution inputs and live authentication. + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::lifecycle::change_run::change_run_in; +use crate::runs::_helpers::requests::permissions::authorize_execution; +use crate::runs::models::{ + CancelRunRequest, RunAuthority, RunChange, RunChangeRequest, RunRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn cancel_person_run( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + request: &CancelRunRequest, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Write).await?, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let Some(source) = rows.into_iter().next() else { + authorize_session( + transaction, + credentials, + &PermissionScope { + action: Action::RunCancel, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + return Err(ControlError::Invalid("run does not exist".into())); + }; + let frozen: RunRequest = serde_json::from_value(source["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, context) = + authorize_execution(transaction, credentials, (&frozen, Action::RunCancel)).await?; + let (transaction, applied) = change_run_in( + transaction, + RunAuthority::Operator(&context), + &RunChangeRequest { + run_id: request.run_id, + event_id: request.event_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: request.reason.clone(), + }, + }, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs index 59cb4f2a40..294eb90d75 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -1,88 +1,22 @@ -//! Retry-safe facts, lifecycle projection and terminal claim release form one transaction. +//! Lifecycle changes share one transaction owner across trusted callers and person operations. -use serde_json::json; - -use crate::credentials::main::append_audit::append_audit; -use crate::runs::_helpers::lifecycle::changes::{ - check_launch_claim, validate_authority, validate_change, validate_retry, -}; -use crate::runs::_helpers::lifecycle::publish_change::publish_change; -use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; -use crate::scheduler::main::check_lease::check_lease; +use crate::runs::_helpers::lifecycle::change_run::change_run_in; +use crate::runs::models::{RunAuthority, RunChangeRequest}; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::decode_row::decode_row; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; -/// Use authenticated authority and record unclaimed cancellation as terminal in the same transaction. pub async fn change_run( store: &ControlStore, authority: RunAuthority<'_>, request: &RunChangeRequest, ) -> Result { - validate_authority(authority, request)?; - let mut transaction = begin(store, TransactionMode::Write).await?; - if let RunAuthority::Scheduler(lease) = authority { - transaction = check_lease(transaction, lease).await?; - } - let body = json!({"change": request}); - let (transaction, previous) = query( - transaction, - "SELECT invocation_id, payload FROM recall_events WHERE event_id=$1 LIMIT 1", - vec![Parameter::Text(request.event_id.to_string())], - ) - .await?; - if let Some(previous) = previous.first() { - validate_retry(previous, request, &body)?; - commit(transaction).await?; - return Ok(false); - } - let (mut transaction, rows) = query( - transaction, - "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ - LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", - vec![Parameter::Id(request.run_id)], + let (transaction, applied) = change_run_in( + begin(store, TransactionMode::Write).await?, + authority, + request, ) .await?; - let state: ChangingRun = decode_row( - rows.into_iter() - .next() - .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, - )?; - if matches!(request.change, RunChange::LaunchFailed { .. }) { - transaction = check_launch_claim(transaction, request).await?; - } else { - validate_change(&state, request)?; - } - let mut transaction = publish_change(transaction, request, &state).await?; - if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = - (authority, &request.change) - { - let mut rows; - (transaction, rows) = query( - transaction, - "SELECT location_id, target, status, cancel_requested_at FROM sqb_runs WHERE run_id=$1", - vec![Parameter::Id(request.run_id)], - ) - .await?; - let after = rows.remove(0); - transaction = append_audit( - transaction, - context, - (request.event_id, "run.cancel_requested"), - ( - json!({"location_id": after["location_id"], "target": after["target"], "run_id": request.run_id}), - json!({"status": state.status, "cancel_requested_at": state.cancel_requested_at}), - json!({"status": after["status"], "cancel_requested_at": after["cancel_requested_at"], "reason": reason}), - ), - ) - .await?; - } - if let RunAuthority::Scheduler(lease) = authority { - transaction = check_lease(transaction, lease).await?; - } commit(transaction).await?; - Ok(true) + Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index e91697d05d..4ca7646a6d 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -1,4 +1,4 @@ -//! Freeze published job inputs for manual requests and sensor evaluations alike. +//! Freeze published jobs for manual and sensor requests in their owning transaction. use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 7fd0c800da..eac7e5ba0e 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -1,4 +1,5 @@ pub mod advance_groups; +pub mod cancel_person_run; pub mod change_run; pub mod claim_next_run; pub(crate) mod frozen_job_request; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs index f44d096434..fbf4bbcc67 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs @@ -1,54 +1,24 @@ -//! Publish a new pinned execution and its operator audit atomically, including retry identity. +//! Trusted local reruns and browser reruns share freezing, idempotence and publication. -use serde_json::json; -use uuid::Uuid; - -use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; -use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; -use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::_helpers::requests::manual_rerun::request_rerun_in; use crate::runs::models::RerunRequest; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; -/// This entry requires direct control-store ownership, just like a privileged manual job request. +/// Omit session credentials only for trusted callers that directly own the control store. pub async fn request_rerun( store: &ControlStore, - context: &AuditContext, + credentials: (&AuditContext, Option<(&str, &str)>), request: &RerunRequest, ) -> Result { - if context.actor.trim().is_empty() - || context.granting_role.trim().is_empty() - || request.reason.trim().is_empty() - { - return Err(ControlError::Invalid( - "rerun needs an actor, granting role and reason".into(), - )); - } - let event_id = Uuid::new_v5(&request.run_id, b"rerun-request"); - let (transaction, retry) = request_retry( + let (transaction, applied) = request_rerun_in( begin(store, TransactionMode::Write).await?, - context, - (request.run_id, event_id, &json!(request)), + credentials, + request, ) .await?; - if retry { - commit(transaction).await?; - return Ok(false); - } - let (transaction, frozen) = frozen_rerun(transaction, context, request).await?; - let (transaction, _) = publish_request(transaction, &frozen).await?; - let transaction = append_audit( - transaction, - context, - (event_id, "run.rerun_requested"), - ( - json!({"location_id": frozen.location_id, "target": frozen.target, "job_key": frozen.job_key, "run_id": request.run_id}), - json!({"source_run_id": request.source_run_id}), - json!({"request": request, "frozen": frozen}), - ), - ).await?; commit(transaction).await?; - Ok(true) + Ok(applied) } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index d63a994176..a2f946e250 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -129,7 +129,7 @@ pub struct JobRunRequest { } /// A retry derives its execution scope from a terminal run, never from caller selectors. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct RerunRequest { pub run_id: Uuid, @@ -139,6 +139,15 @@ pub struct RerunRequest { pub priority: i64, } +/// Browser cancellation cannot supply an actor, claim or lifecycle transition. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct CancelRunRequest { + pub run_id: Uuid, + pub event_id: Uuid, + pub reason: String, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct RunRecord { pub run_id: Uuid, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index 1c5c1a8681..c30f34ce3e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -16,12 +16,16 @@ use crate::credentials::models::{ }; use crate::credentials::tests::location_helpers::create_location; use crate::runs::main::{ - list_person_runs::list_person_runs, list_runs::list_runs, - request_person_job_run::request_person_job_run, + advance_groups::advance_groups, cancel_person_run::cancel_person_run, change_run::change_run, + list_person_runs::list_person_runs, list_runs::list_runs, request_job_run::request_job_run, + request_person_job_run::request_person_job_run, request_rerun::request_rerun, }; -use crate::runs::models::JobRunRequest; -use crate::runs::tests::helpers::snapshot; +use crate::runs::models::{ + CancelRunRequest, JobRunRequest, RerunRequest, RunAuthority, RunChange, RunChangeRequest, +}; +use crate::runs::tests::helpers::{request_run, snapshot}; use crate::runs::tests::test_types::PersonRunPermissionTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; @@ -124,7 +128,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi "customers": {"description":"Build customers", "selection":["customers"], "settings":inputs["job_settings"][0]} })); input.documents.insert("groups".into(), json!({"orders_group": {"description":"Build both jobs", "selection":["customers","orders"], "jobs":{"daily_orders":[],"customers":[]}}})); - publish_snapshot(store, &input) + let published = publish_snapshot(store, &input) .await .expect("publish immutable job"); let request = JobRunRequest { @@ -173,6 +177,11 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi .expect("built-in metadata access") .is_empty() ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut reruns = Vec::new(); for job_key in ["daily_orders", "orders_group"] { let result = request_person_job_run( store, @@ -196,14 +205,157 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi "{}", test_case.description ); + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + job_key: job_key.into(), + ..request.clone() + }, + ) + .await + .expect("owner creates source with frozen settings"); + let cancellation = CancelRunRequest { + run_id: source_id, + event_id: Uuid::now_v7(), + reason: "Stop before worker launch".into(), + }; + let can_cancel = test_case.role != BuiltinRole::Viewer; + for expected in [true, false] { + assert_eq!( + cancel_person_run(store, (&context, credentials), &cancellation) + .await + .ok(), + can_cancel.then_some(expected), + "cancellation permits stopping even sensitive work" + ); + } + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + job_key: job_key.into(), + ..request.clone() + }, + ) + .await + .expect("owner creates independent rerun source"); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: source_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: cancellation.reason.clone(), + }, + }, + ) + .await + .expect("owner stops independent rerun source"); + for _ in 0..2 { + advance_groups( + store, + &lease, + (&Default::default(), (location_id, "development")), + ) + .await + .expect("scheduler confirms stopped children and parent"); + } + let rerun = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_id, + from_failure: false, + reason: "Rerun the original frozen execution".into(), + priority: 0, + }; + for expected in [true, false] { + let result = + request_rerun(store, (&context, Some(credentials)), &rerun).await; + assert_eq!( + result.as_ref().ok().copied(), + test_case.expected_allowed.then_some(expected), + "{}: frozen job/group rerun and idempotent retry: {result:?}", + test_case.description, + ); + assert_eq!( + matches!(result, Err(ControlError::Denied { .. })), + !test_case.expected_allowed, + "denial must be authorization, not invalid input" + ); + } + assert!(matches!( + request_rerun( + store, + (&context, Some((credentials.0, "wrong-csrf"))), + &rerun + ) + .await, + Err(ControlError::Denied { .. }) + )); + reruns.push(rerun); } + let mut adhoc = crate::runs::tests::helpers::request( + (location_id, published.snapshot_id), + Uuid::now_v7().as_u128(), + 0, + &["orders"], + ); + adhoc.settings = + serde_json::from_value(inputs["job_settings"][settings_index].clone()) + .expect("ad-hoc frozen settings"); + request_run(store, &adhoc) + .await + .expect("owner requests ad-hoc execution"); + cancel_person_run( + store, + ( + &context, + (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()), + ), + &CancelRunRequest { + run_id: adhoc.run_id, + event_id: Uuid::now_v7(), + reason: "Stop ad-hoc source".into(), + }, + ) + .await + .expect("administrator stops ad-hoc execution"); + let adhoc_rerun = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: adhoc.run_id, + from_failure: false, + reason: "Repeat ad-hoc execution".into(), + priority: 0, + }; + let can_rerun_adhoc = test_case.role == BuiltinRole::Admin + || (test_case.role == BuiltinRole::Developer && settings_index == 0); + let result = + request_rerun(store, (&context, Some(credentials)), &adhoc_rerun).await; + assert_eq!( + result.as_ref().ok().copied(), + can_rerun_adhoc.then_some(true), + "ad-hoc: {result:?}" + ); assert_eq!( - list_runs(store, (location_id, "development"), 10) + matches!(result, Err(ControlError::Denied { .. })), + !can_rerun_adhoc + ); + reruns.push(adhoc_rerun); + assert_eq!( + list_runs(store, (location_id, "development"), 100) .await .expect("queue projection") .len(), - 4 * usize::from(test_case.expected_allowed), - "denial cannot publish work; success queues the job, group and two children" + 9 + 8 * usize::from(test_case.expected_allowed) + + usize::from(can_rerun_adhoc), + "owner sources plus allowed manual/rerun jobs and expanded groups only" ); let administrator_credentials = (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()); @@ -254,6 +406,15 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi ), "reenabling cannot restore a revoked grant" ); + for rerun in reruns { + assert!( + matches!( + request_rerun(store, (&context, Some(credentials)), &rerun).await, + Err(ControlError::Denied { .. }) + ), + "retry must reauthorize after revocation" + ); + } close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 75339ea1fe..cc08da0422 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -93,7 +93,9 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ priority: 7, }; assert!( - request_rerun(store, &context, &retry).await.is_err(), + request_rerun(store, (&context, None), &retry) + .await + .is_err(), "a live source cannot be rerun" ); let lease = acquire_lease(store, Uuid::now_v7()) @@ -192,7 +194,11 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; install_fault(store, &fault).await; - assert!(request_rerun(store, &context, &retry).await.is_err()); + assert!( + request_rerun(store, (&context, None), &retry) + .await + .is_err() + ); remove_fault(store, &fault).await; assert_eq!( list_runs(store, scope, 20) @@ -203,20 +209,24 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ "audit failure rolls back parent and all children" ); let (first, again) = tokio::join!( - request_rerun(store, &context, &retry), - request_rerun(store, &context, &retry) + request_rerun(store, (&context, None), &retry), + request_rerun(store, (&context, None), &retry) ); assert_ne!(first.expect("rerun"), again.expect("replica retry")); let changed = RerunRequest { from_failure: false, ..retry.clone() }; - assert!(request_rerun(store, &context, &changed).await.is_err()); + assert!( + request_rerun(store, (&context, None), &changed) + .await + .is_err() + ); let other = AuditContext { actor: "other-operator".into(), ..context.clone() }; - assert!(request_rerun(store, &other, &retry).await.is_err()); + assert!(request_rerun(store, (&other, None), &retry).await.is_err()); let settings = BTreeMap::from([ ( "staging", @@ -249,7 +259,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ] { let request = execution.request; assert_eq!( - request_rerun(store, &context, &request) + request_rerun(store, (&context, None), &request) .await .expect("publish rerun"), execution.expected_applied, @@ -311,7 +321,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ); assert!(slot_counts(store).await.is_empty()); assert!( - !request_rerun(store, &context, &request) + !request_rerun(store, (&context, None), &request) .await .expect("completed transport retry") ); @@ -322,7 +332,9 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ..request.clone() }; assert!( - request_rerun(store, &context, &successful).await.is_err(), + request_rerun(store, (&context, None), &successful) + .await + .is_err(), "successful work has nothing to retry from failure" ); } diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 2cc01e3a1c..6a6ce0a5fe 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -340,12 +340,12 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r ip_address: None, }; assert!( - request_rerun(store, &context, &retry) + request_rerun(store, (&context, None), &retry) .await .expect("request rerun") ); assert!( - !request_rerun(store, &context, &retry) + !request_rerun(store, (&context, None), &retry) .await .expect("retry request") ); diff --git a/deploy/README.md b/deploy/README.md index 5edd137adf..410ac19c79 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -147,10 +147,18 @@ also works while a person is disabled, before they are enabled again. Password sessions use `GET /api/control/locations/{location_id}/targets/{target}/runs` for recent run metadata, `POST /api/control/runs` for published-job requests, and -`POST /api/control/triggers` for trigger overrides. Viewers can read metadata but cannot -run jobs; operators and developers can run ordinary jobs and toggle triggers. Full-refresh -jobs, published cursor bounds, explicit backfill windows and cursor resets require an installation -administrator in these endpoints. The server defaults to `--default-role none`: a person with no +`POST /api/control/triggers` for trigger overrides. Use `POST /api/control/runs/cancel` +with `run_id`, `event_id` and `reason` to request cancellation; use +`POST /api/control/runs/rerun` with `run_id`, `source_run_id`, `from_failure`, `reason` +and `priority` to repeat a terminal run. Retrying an identical request is idempotent, but +still requires live permission. Reruns keep the original snapshot, image, selection and bounds; +`from_failure` narrows selection to durable failed/skipped outcomes. Cancellation of active +work remains a request until its worker confirms shutdown; it never releases ownership early. +Viewers can read metadata but cannot run, cancel or rerun jobs; operators and developers can +control ordinary jobs and toggle triggers. Ad-hoc reruns also require developer or admin access. +Full-refresh jobs, published cursor bounds, explicit backfill windows and their reruns require +an installation administrator in these endpoints. Sensitive work can still be cancelled without +history-rewrite permission. The server defaults to `--default-role none`: a person with no assigned role cannot use them. Set `--default-role viewer` to give every authenticated person installation-wide metadata access, including people whose explicit roles have been revoked. This is a live server setting, not a persisted assignment; returning it to `none` removes that diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index 17a425ef1d..ee4242c6e0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -14,6 +14,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( PersonJourneyTestCase, + WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( current_snapshot, @@ -96,6 +97,18 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi "location_id": worker.api.location_id, "target": "development", } + rerun: dict[str, Any] = { + "run_id": str(uuid4()), + "source_run_id": scheduled["run_id"], + "from_failure": False, + "reason": "Repeat the pinned order build", + "priority": 0, + } + cancellation: dict[str, Any] = { + "run_id": scheduled["run_id"], + "event_id": str(uuid4()), + "reason": "Stop the order build", + } with test_case.expected_metadata_access(): assert ( person_read(worker.api, runs_path, session=session)[0]["run_id"] @@ -104,6 +117,8 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi for path, body in [ ("/api/control/runs", request), ("/api/admin/roles", assignment), + ("/api/control/runs/cancel", cancellation), + ("/api/control/runs/rerun", rerun), ]: with pytest.raises(HTTPError) as rejected: person_request(worker.api, path, body=body, session=session) @@ -117,6 +132,8 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi assert not ({"sql", "compiled_sql", "output", "rows", "evidence"} & metadata[0].keys()) for path, body in [ ("/api/control/runs", request), + ("/api/control/runs/cancel", cancellation), + ("/api/control/runs/rerun", rerun), ("/api/admin/people/" + admin["principal_id"], "disable"), ]: with pytest.raises(HTTPError) as rejected: @@ -221,11 +238,160 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi assert person_request( worker.api, "/api/admin/roles", body=assignment, session=admin_session ) == {"applied": True} - with pytest.raises(HTTPError) as revoked: + for path, body in [ + ("/api/control/runs", request | {"run_id": str(uuid4())}), + ("/api/control/runs/cancel", cancellation), + ("/api/control/runs/rerun", rerun), + ]: + with pytest.raises(HTTPError) as revoked: + person_request(worker.api, path, body=body, session=session) + assert revoked.value.code == 403 + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="scoped operator frozen rerun, cancellation and revoked-state privacy", + model_sql="SELECT 23 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_person_run_controls_when_scopes_change_then_frozen_execution_and_state_privacy_hold( + test_case: WorkerExecutionTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + scheduled: dict[str, Any] = worker_result(worker, test_case) + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-person" + ) + admin_session: tuple[str, str] = person_login(worker.api, admin["name"]) + session: tuple[str, str] = person_login(worker.api, person["name"]) + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "operator", + "location_id": worker.api.location_id, + "target": "preview", + } + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + rerun: dict[str, Any] = { + "run_id": str(uuid4()), + "source_run_id": scheduled["run_id"], + "from_failure": False, + "reason": "Repeat pinned order build", + "priority": 0, + } + for from_failure in [False, True]: + with pytest.raises(HTTPError) as out_of_scope: + person_request( + worker.api, + "/api/control/runs/rerun", + body=rerun | {"from_failure": from_failure}, + session=session, + ) + assert out_of_scope.value.code == 403 + assignment["target"] = "development" + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + for applied in [True, False]: + assert person_request( + worker.api, "/api/control/runs/rerun", body=rerun, session=session + ) == {"applied": applied} + repeated: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=rerun["run_id"] + ) + assert repeated["status"] == test_case.expected_status + assert repeated["snapshot_id"] == scheduled["snapshot_id"] + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + worker.process.terminate() + assert worker.process.wait(timeout=5) == 0 + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": worker.api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Build order totals", + "priority": 0, + } + assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { + "applied": True + } + cancellation: dict[str, Any] = { + "run_id": request["run_id"], + "event_id": str(uuid4()), + "reason": "Stop before launch", + } + with pytest.raises(HTTPError) as forged: person_request( worker.api, - "/api/control/runs", - body=request | {"run_id": str(uuid4())}, + "/api/control/runs/cancel", + body=cancellation | {"actor": "local-control-store-owner"}, session=session, ) - assert revoked.value.code == 403 + assert forged.value.code == 422 + with pytest.raises(HTTPError) as csrf: + person_request( + worker.api, + "/api/control/runs/cancel", + body=cancellation, + session=(session[0], "wrong-csrf"), + ) + assert csrf.value.code == 401 + for applied in [True, False]: + assert person_request( + worker.api, "/api/control/runs/cancel", body=cancellation, session=session + ) == {"applied": applied} + runs_path: str = f"/api/control/locations/{worker.api.location_id}/targets/development/runs" + assert person_read(worker.api, runs_path, session=session)[0]["status"] == "cancelled" + assignment["revoke"] = True + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + queued: dict[str, Any] = request | {"run_id": str(uuid4())} + assert person_request( + worker.api, "/api/control/runs", body=queued, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as metadata: + person_read(worker.api, runs_path, session=session) + assert metadata.value.code == 403 + for source_id in [scheduled["run_id"], request["run_id"], queued["run_id"], str(uuid4())]: + for from_failure in [False, True]: + with pytest.raises(HTTPError) as private: + person_request( + worker.api, + "/api/control/runs/rerun", + body=rerun + | { + "run_id": str(uuid4()), + "source_run_id": source_id, + "from_failure": from_failure, + }, + session=session, + ) + assert private.value.code == 403 + with pytest.raises(HTTPError) as cancel_denied: + person_request( + worker.api, + "/api/control/runs/cancel", + body=cancellation | {"run_id": source_id}, + session=session, + ) + assert cancel_denied.value.code == 403 From 28499ad68b3115bce608b3b7e0ae1aa39f9a88ac Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 04:03:31 +0000 Subject: [PATCH 082/138] test: exercise offline administrator recovery in installed wheels --- .github/workflows/build-distributions.yml | 5 ++ .github/workflows/verify.yml | 12 +-- .../publication/_test_types.py | 7 ++ .../publication/test_installed_recovery.py | 32 +++++++ .../publication/wheel/api/helpers.py | 44 ++++++++++ .../publication/wheel/cli/helpers.py | 77 ++++++++++++++++ .../publication/wheel/helpers.py | 87 +++++++++++++++++++ 7 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index ce6ed51bfb..e2c5b8e127 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -189,6 +189,11 @@ jobs: "${run_as[@]}" "${smoke_scripts}/sqb" --version "${run_as[@]}" "${smoke_scripts}/sqb" --help "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help + if [[ "${{ inputs.prebuilt-run-id == '' }}" == "true" ]]; then + cd "${GITHUB_WORKSPACE}" + "${run_as[@]}" "${smoke_python}" -m tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers \ + "${smoke_scripts}/sqb${{ runner.os == 'Windows' && '.exe' || '' }}" + fi - name: Check wheel contents and run a native command if: matrix.platform.name == 'linux-x86_64' run: | diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 9df9458535..8f4426a27d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -249,6 +249,8 @@ jobs: run: uv python install 3.12 - name: Verify lockfile run: uv lock --check --python 3.12 + - name: Verify native workspace lockfile + run: cargo metadata --locked --format-version 1 > /dev/null checks: name: Static checks and unit + integration tests @@ -471,15 +473,7 @@ jobs: shared-key: verify-windows save-if: false - name: Test the native discovery crate (walk, junctions, ordering, read errors) - run: | - status=0 - cargo test --locked -p sqlbuild-discovery || status=$? - if [ "${status}" -ne 0 ]; then - echo "Windows dependency resolution diagnostics (original failure is retained):" - cargo tree --offline -p sqlbuild-discovery || true - git diff -- Cargo.lock - fi - exit "${status}" + run: cargo test --locked -p sqlbuild-discovery - name: Install Python run: uv python install 3.12 - name: Install SQLBuild and pytest diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 0b58b14ab8..34aa67f911 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -198,3 +198,10 @@ class DevelopmentTestCase: class ComposeDeploymentTestCase: description: str expected_warehouse_rows: list[list[int]] + + +@dataclass(frozen=True) +class InstalledRecoveryTestCase: + description: str + executable: str + expected_rejection: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py new file mode 100644 index 0000000000..69f482a87b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py @@ -0,0 +1,32 @@ +"""An installed native executable retains offline administrator recovery guarantees.""" + +import shutil +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + InstalledRecoveryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers import ( + verify_recovery, +) + + +@pytest.mark.parametrize( + "test_case", + [ + InstalledRecoveryTestCase( + description="installed native server recovers authority without rebootstrap", + executable="sqb", + expected_rejection="cannot remove or disable the final usable installation administrator", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_installed_server_when_recovered_offline_then_authority_is_restored_without_rebootstrap( + test_case: InstalledRecoveryTestCase, +) -> None: + binary: str | None = shutil.which(test_case.executable) + assert binary is not None + assert test_case.expected_rejection in verify_recovery(Path(binary)) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py new file mode 100644 index 0000000000..020788e0e5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py @@ -0,0 +1,44 @@ +"""Exercise installed-server browser sessions over a disposable local HTTP transport.""" + +from __future__ import annotations + +import json +from email.message import Message +from http.client import HTTPResponse +from http.cookies import SimpleCookie +from typing import Any +from urllib.error import HTTPError +from urllib.request import Request, urlopen + + +def request( + endpoint: str, body: bytes | None = None, session: tuple[str, str] = ("", "") +) -> tuple[int | None, Message[str, str], dict[str, Any]]: + """Return denials as ordinary results; never print credentials or response payloads.""" + headers: dict[str, str] = { + "Host": "control.example.com", + "Origin": "https://control.example.com", + "Content-Type": "application/json", + "Cookie": session[0], + "x-sqb-csrf": session[1], + } + call: Request = Request(endpoint, data=body, headers=headers) + response: HTTPResponse | HTTPError + try: + response = urlopen(call, timeout=5) + except HTTPError as error: + response = error + with response: + return response.status, response.headers, json.load(response) + + +def login(url: str, username: str, password: str) -> tuple[str, str]: + """Authenticate with the real Argon2/password route and retain its host-only cookie.""" + status, headers, body = request( + f"{url}/api/auth/password/login", + json.dumps({"username": username, "password": password}).encode("utf-8"), + ) + assert status == 200, f"Password sign-in failed with HTTP {status}" + cookies: SimpleCookie[str] = SimpleCookie("\n".join(headers.get_all("Set-Cookie", []))) + cookie: str = "__Host-sqb-session=" + cookies["__Host-sqb-session"].value + return cookie, body["csrf_token"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py new file mode 100644 index 0000000000..f9be256afb --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py @@ -0,0 +1,77 @@ +"""Start and reap an installed native executable in a disposable installation.""" + +import json +import os +import subprocess +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + + +def environment() -> dict[str, str]: + """Never inherit a real control store into a disposable acceptance run.""" + values: dict[str, str] = os.environ.copy() + values.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + return values + + +def administer( + installation: tuple[Path, Path], arguments: list[str], password: str | None = None +) -> subprocess.CompletedProcess[str]: + """Password input remains on stdin and is never included in process arguments or logs.""" + binary, root = installation + return subprocess.run( + [str(binary), "admin", *arguments], + cwd=root, + env=environment(), + input=password, + text=True, + encoding="utf-8", + capture_output=True, + timeout=20, + check=False, + ) + + +def successful(output: subprocess.CompletedProcess[str]) -> dict[str, Any]: + """Parse machine stdout only after the native command completed successfully.""" + assert output.returncode == 0, f"Native administration failed: {output.stderr}" + return json.loads(output.stdout) + + +@contextmanager +def serve(installation: tuple[Path, Path]) -> Iterator[str]: + """Kill and reap the server before returning, on Windows as well as Unix.""" + binary, root = installation + log: Path = root / "server.log" + with log.open("w", encoding="utf-8") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [ + str(binary), + "server", + "--port", + "0", + "--no-scheduler", + "--password-origin", + "https://control.example.com", + ], + cwd=root, + env=environment(), + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + deadline: float = time.monotonic() + 10 + while "Control API ready: " not in log.read_text(encoding="utf-8"): + assert process.poll() is None, log.read_text(encoding="utf-8") + assert time.monotonic() < deadline, log.read_text(encoding="utf-8") + time.sleep(0.02) + address: str = ( + log.read_text(encoding="utf-8").split("Control API ready: ", 1)[1].splitlines()[0] + ) + yield f"http://{address}" + finally: + process.kill() + process.wait(timeout=10) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py new file mode 100644 index 0000000000..5d2df6f246 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py @@ -0,0 +1,87 @@ +"""Bootstrap, authenticate, lose authority, recover offline and authenticate again.""" + +import sqlite3 +import subprocess +import sys +from contextlib import closing +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.api.helpers import ( + login, + request, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + serve, + successful, +) + + +def verify_recovery(binary: Path) -> str: + """Exercise the wheel's native executable using only the Python standard library.""" + binary = binary.resolve(strict=True) + print("Starting installed-artifact administrator recovery acceptance...", flush=True) + with TemporaryDirectory(prefix="sqlbuild-wheel-recovery-") as temporary: + root: Path = Path(temporary) + installation: tuple[Path, Path] = binary, root + person: dict[str, Any] = successful( + administer(installation, ["create-user", "orders-admin"]) + ) + principal: str = person["principal_id"] + password = "orders-recovery-password-2026" + successful(administer(installation, ["set-password", principal], password)) + recovery: list[str] = [ + "recover-user", + principal, + "--reason", + "restore local administrator access", + ] + with serve(installation) as url: + previous: tuple[str, str] = login(url, person["name"], password) + rejected: subprocess.CompletedProcess[str] = administer(installation, recovery) + assert rejected.returncode != 0 and not rejected.stdout + assert "exclusive access" in rejected.stderr + print("Online recovery rejected; server killed and reaped.", flush=True) + database: Path = root / ".sqlbuild/control.sqlite3" + with closing(sqlite3.connect(database)) as control: + control.execute("UPDATE sqb_principals SET disabled_at='2026-01-01T00:00:00Z'") + control.commit() + rejected = administer(installation, ["create-user", "replacement-admin"]) + assert rejected.returncode != 0 and not rejected.stdout + assert "bootstrap is closed" in rejected.stderr + result: dict[str, Any] = successful(administer(installation, recovery)) + assert result == {"principal_id": principal, "role": "admin", "recovered": True} + with closing(sqlite3.connect(database)) as control: + assert control.execute("SELECT COUNT(*) FROM sqb_admin_initialization").fetchone() == ( + 1, + ) + assert control.execute( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.offline_recovered'" + ).fetchone() == (1,) + assert control.execute( + "SELECT COUNT(*) FROM sqb_password_sessions WHERE revoked_at IS NULL" + ).fetchone() == (0,) + print( + "Offline recovery committed one audit; bootstrap stayed closed; sessions revoked.", + flush=True, + ) + with serve(installation) as url: + assert request(f"{url}/api/auth/session", session=previous)[0] == 401 + session: tuple[str, str] = login(url, person["name"], password) + assert request(f"{url}/api/auth/session", session=session)[0] == 200 + status, _, body = request(f"{url}/api/admin/people/{principal}", b'"revoke"', session) + assert status == 403 + assert ( + "cannot remove or disable the final usable installation administrator" + in body["error"] + ) + rejected = administer(installation, ["create-user", "replacement-admin"]) + assert rejected.returncode != 0 and "bootstrap is closed" in rejected.stderr + print("Installed-artifact administrator recovery acceptance passed.", flush=True) + return body["error"] + + +if __name__ == "__main__": + verify_recovery(Path(sys.argv[1])) From 1e9b86e09dcad1828f78211d86b76fddceb64833 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 04:55:29 +0000 Subject: [PATCH 083/138] feat: issue federated browser sessions with live person authorization --- .../orchestration/proxy_identities.json | 6 +- clients/typescript/src/schema.d.ts | 17 +- .../tests/native_commands/test_proxy.rs | 18 +- .../api/_helpers/authentication/configure.rs | 11 +- .../src/api/_helpers/authentication/oidc.rs | 17 +- .../authentication/password_sessions.rs | 63 +-- .../authentication/password_transport.rs | 41 +- .../_helpers/authentication/trusted_proxy.rs | 34 +- .../api/_helpers/resources/administrators.rs | 22 +- .../sqlbuild-orchestration/src/api/models.rs | 16 +- .../_helpers/administrator_changes.rs | 6 +- .../_helpers/administrator_roles.rs | 5 +- .../authorization/_helpers/administrators.rs | 24 +- .../authorization/main/assign_builtin_role.rs | 6 +- .../main/change_administrator.rs | 17 +- .../main/recover_administrator.rs | 9 +- .../credentials/_helpers/browser_sessions.rs | 67 ++++ .../src/credentials/_helpers/mod.rs | 1 + .../credentials/_helpers/password_sessions.rs | 49 +-- .../main/authenticate_bound_person.rs | 10 +- .../main/authenticate_session_in.rs | 11 +- .../src/credentials/main/revoke_session.rs | 83 ++-- .../src/credentials/main/set_password.rs | 2 + .../src/credentials/models.rs | 8 + .../src/store/_helpers/migration.rs | 1 + .../src/store/external_sessions.sql | 10 + .../src/store/main/migrate_store.rs | 7 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_external_sessions.rs | 379 ++++++++++++++++++ .../tests/api/test_location_work.rs | 2 +- .../tests/api/test_oidc.rs | 81 +++- .../tests/api/test_proxy.rs | 101 ++++- .../tests/api/test_types.rs | 6 + .../tests/api/warehouse_metadata/helpers.rs | 3 +- .../credentials/sessions/test_contract.rs | 47 ++- .../tests/credentials/test_administrators.rs | 102 ++++- .../tests/credentials/test_recovery.rs | 30 +- .../tests/runs/test_permissions.rs | 22 +- .../tests/sensors/test_runs.rs | 1 + .../tests/snapshots/helpers.rs | 2 +- deploy/README.md | 24 +- fensu.toml | 4 +- 42 files changed, 1122 insertions(+), 245 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs create mode 100644 crates/sqlbuild-orchestration/src/store/external_sessions.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json b/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json index 2f75160f35..ddbe5d106b 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json +++ b/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json @@ -5,5 +5,9 @@ "service_subject": "external-orders-service", "absent_subject": "external-absent-reader", "unknown_subject": "external-unbound-reader", - "untrusted_peer": "192.0.2.24" + "untrusted_peer": "192.0.2.24", + "administrator_journey": { + "bound": ["orders-admin", "customers-admin"], + "unbound_password_person": "products-admin" + } } diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index dfa038206e..bed94a0536 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -55,7 +55,7 @@ export interface paths { path?: never; cookie?: never; }; - /** @description Validates one browser-bound code and returns the explicitly bound person only; no session, bearer, provisioning or grants. */ + /** @description Validates one browser-bound code and issues a bounded browser session for an explicitly bound person; no provisioning, personal API tokens or implicit grants. */ get: operations["oidcIdentity"]; put?: never; post?: never; @@ -121,7 +121,7 @@ export interface paths { path?: never; cookie?: never; }; - /** @description Requires a configured socket peer and an upstream-replaced X-SQB-User subject. Does not issue sessions or grant permissions. */ + /** @description Requires a configured socket peer and upstream-replaced X-SQB-User subject; issues a bounded browser session without provisioning, personal API tokens or implicit grants. */ get: operations["proxyIdentity"]; put?: never; post?: never; @@ -621,8 +621,13 @@ export interface components { status: "timed_out"; }; ExternalIdentityResponse: { + csrf_token: string; + /** Format: date-time */ + expires_at: string; /** Format: uuid */ principal_id: string; + /** Format: uuid */ + session_id: string; }; /** @description Fingerprint observations from one registered reader, independent of automatic polling. */ FingerprintBatch: { @@ -1560,6 +1565,14 @@ export interface operations { "application/json": components["schemas"]["AuthenticationDenial"]; }; }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; }; }; currentSession: { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs index 1afdbe36a0..e5a5119fcf 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs @@ -37,8 +37,22 @@ fn given_an_owner_configured_proxy_when_a_standalone_server_authenticates_then_o .header(header::HOST, "control.example.com").header(header::ORIGIN, "https://control.example.com") .header("x-sqb-user", test_case.subject).send().await.expect("upstream identity"); assert_eq!(response.status(), test_case.expected_status, "{}", test_case.description); - assert!(!response.headers().contains_key(header::SET_COOKIE)); - assert_eq!(response.json::().await.expect("identity"), json!({"principal_id": principal})); + let cookie = response.headers().get_all(header::SET_COOKIE).iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")).expect("browser session") + .split(';').next().expect("cookie pair").to_owned(); + let body = response.json::().await.expect("identity"); + assert_eq!(body["principal_id"], principal); + assert!(body.get("bearer").is_none()); + let session = client.get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com").header(header::COOKIE, &cookie) + .send().await.expect("standalone shared session path"); + assert_eq!(session.status(), StatusCode::OK); + assert_eq!(session.json::().await.expect("session identity")["principal_id"], principal); + let metadata = client.get(format!("{url}/api/control/locations/{}/targets/development/runs", uuid::Uuid::now_v7())) + .header(header::HOST, "control.example.com").header(header::COOKIE, &cookie) + .send().await.expect("no implicit viewer grant"); + assert_eq!(metadata.status(), StatusCode::FORBIDDEN); let machine = client.get(format!("{url}/api/locations/{}/health", uuid::Uuid::now_v7())) .header("x-sqb-user", test_case.subject).send().await.expect("machine boundary"); assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs index a39792d05e..10897b0992 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs @@ -6,6 +6,7 @@ use axum::{Extension, Router}; use crate::api::_helpers::authentication::{oidc, password_sessions, trusted_proxy}; use crate::api::models::{ApiAuthentication, DevelopmentScope}; +use crate::credentials::models::AdministratorAuthentication; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -43,11 +44,19 @@ pub(crate) async fn configure( "OIDC authentication cannot be combined with other human authentication modes".into(), )); } + let administrator_authentication = if let Some(settings) = &trusted_proxy { + AdministratorAuthentication::BoundPeople(settings.identities.values().copied().collect()) + } else if let Some(settings) = &oidc { + AdministratorAuthentication::BoundPeople(settings.identities.values().copied().collect()) + } else { + AdministratorAuthentication::Password + }; Ok(( development_scope, password_sessions::router(store, password_origin)? .merge(trusted_proxy::router(store, trusted_proxy)?) .merge(oidc::router(store, oidc).await?) - .layer(Extension(default_role)), + .layer(Extension(default_role)) + .layer(Extension(administrator_authentication)), )) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs index 80055a7050..c17062cb61 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs @@ -6,14 +6,14 @@ use axum::extract::{Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Redirect, Response}; use axum::routing::get; -use axum::{Extension, Json, Router, middleware}; +use axum::{Extension, Router, middleware}; use openidconnect::core::CoreAuthenticationFlow; use openidconnect::{CsrfToken, Nonce, PkceCodeChallenge}; -use crate::api::_helpers::authentication::oidc_provider; use crate::api::_helpers::authentication::password_transport::{ - browser_cookie, deny, guard, set_cookie, + browser_cookie, deny, external_session, guard, set_cookie, }; +use crate::api::_helpers::authentication::{oidc_provider, password_sessions}; use crate::api::constants::OIDC_FLOW_COOKIE; use crate::api::errors::ApiError; use crate::api::models::{ @@ -44,6 +44,7 @@ pub(crate) async fn router( Ok(Router::new() .route("/api/auth/oidc/login", get(login)) .route("/api/auth/oidc/callback", get(callback)) + .merge(password_sessions::protected(store)) .layer(middleware::from_fn_with_state(store.clone(), guard)) .layer(Extension(BrowserOrigin(runtime.settings.origin.clone()))) .layer(Extension(Arc::new(runtime)))) @@ -96,7 +97,7 @@ pub(crate) async fn login( } #[utoipa::path(get, path = "/api/auth/oidc/callback", operation_id = "oidcIdentity", - description = "Validates one browser-bound code and returns the explicitly bound person only; no session, bearer, provisioning or grants.", + description = "Validates one browser-bound code and issues a bounded browser session for an explicitly bound person; no provisioning, personal API tokens or implicit grants.", responses((status = 200, body = ExternalIdentityResponse), (status = 401, body = AuthenticationDenial)))] pub(crate) async fn callback( State(store): State, @@ -129,19 +130,15 @@ pub(crate) async fn callback( let mut response = match subject { Ok(subject) => match runtime.settings.identities.get(&subject).copied() { Some(principal) => { - if authenticate_bound_person( + if let Some(session) = authenticate_bound_person( &store, &context, principal, ExternalIdentityMechanism::Oidc, ) .await? - .is_some() { - Json(ExternalIdentityResponse { - principal_id: principal, - }) - .into_response() + external_session(session)? } else { deny(&store, &context, "inactive_oidc_person").await? } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index cd15d8496b..16f92c1751 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -7,14 +7,14 @@ use axum::routing::{get, post}; use axum::{Extension, Json, Router, middleware}; use crate::api::_helpers::authentication::password_transport::{ - authenticate, deny, guard, parse_origin, set_cookie, + authenticate, deny, guard, parse_origin, session_cookies, set_cookie, }; use crate::api::constants::{PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE}; use crate::api::errors::ApiError; use crate::api::models::{ AuthenticationDenial, PasswordChangeRequest, PasswordLoginRequest, PasswordSessionResponse, }; -use crate::credentials::constants::{LOGIN_WINDOW_SECONDS, PASSWORD_SESSION_SECONDS}; +use crate::credentials::constants::LOGIN_WINDOW_SECONDS; use crate::credentials::main::{ login_password::login_password, reserve_authentication_attempt::reserve_authentication_attempt, revoke_session::revoke_session, set_password::set_password, verify_password::verify_password, @@ -23,20 +23,13 @@ use crate::credentials::models::{ AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, SessionIdentity, }; use crate::store::errors::ControlError; -use crate::store::models::ControlStore; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; -pub(crate) fn router( - store: &ControlStore, - origin: Option, -) -> Result, ControlError> { - let Some(origin) = origin else { - return Ok(Router::new()); - }; - let origin = parse_origin(&origin, "password")?; - let protected = Router::new() +pub(crate) fn protected(store: &ControlStore) -> Router { + Router::new() .route("/api/auth/session", get(current)) .route("/api/auth/logout", post(logout)) - .route("/api/auth/password", post(change_password)) .route( "/api/admin/people/{principal_id}", post(crate::api::_helpers::resources::administrators::change), @@ -65,7 +58,23 @@ pub(crate) fn router( "/api/control/locations/{location_id}/targets/{target}/runs", get(crate::api::_helpers::resources::person_operations::recent_runs), ) - .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) +} + +pub(crate) fn router( + store: &ControlStore, + origin: Option, +) -> Result, ControlError> { + let Some(origin) = origin else { + return Ok(Router::new()); + }; + let origin = parse_origin(&origin, "password")?; + let protected = protected(store).merge( + Router::new() + .route("/api/auth/password", post(change_password)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)), + ); Ok(protected .route("/api/auth/password/login", post(login)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) @@ -84,24 +93,12 @@ pub(crate) async fn login( ) -> Result { match login_password(&store, &context, (&request.username, &request.password)).await? { PasswordLogin::Authenticated(session) => { - let mut response = Json(PasswordSessionResponse { - identity: session.identity, + let response = Json(PasswordSessionResponse { + identity: session.identity.clone(), csrf_token: session.csrf.clone(), }) .into_response(); - set_cookie( - &mut response, - (PASSWORD_COOKIE, &session.bearer), - PASSWORD_SESSION_SECONDS, - (true, "Strict"), - )?; - set_cookie( - &mut response, - (PASSWORD_CSRF_COOKIE, &session.csrf), - PASSWORD_SESSION_SECONDS, - (false, "Strict"), - )?; - Ok(response) + session_cookies(response, &session) } PasswordLogin::Denied { decision_id, @@ -150,7 +147,13 @@ pub(crate) async fn logout( Extension(context): Extension, Extension(identity): Extension, ) -> Result { - revoke_session(&store, &context, &identity).await?; + let (transaction, _) = revoke_session( + begin(&store, TransactionMode::Write).await?, + &context, + (identity.principal_id, Some(identity.session_id)), + ) + .await?; + commit(transaction).await?; clear_cookies() } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs index 68256046ad..9927f449cb 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -10,12 +10,15 @@ use axum::{Extension, Json}; use serde_json::json; use uuid::Uuid; -use crate::api::constants::{HTTPS_SCHEME, PASSWORD_COOKIE, PASSWORD_CSRF_HEADER, ROOT_PATH}; +use crate::api::constants::{ + HTTPS_SCHEME, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE, PASSWORD_CSRF_HEADER, ROOT_PATH, +}; use crate::api::errors::ApiError; -use crate::api::models::{AuthenticationDenial, BrowserOrigin}; +use crate::api::models::{AuthenticationDenial, BrowserOrigin, ExternalIdentityResponse}; +use crate::credentials::constants::PASSWORD_SESSION_SECONDS; use crate::credentials::main::authenticate_session::authenticate_session; use crate::credentials::main::record_authentication_denial::record_authentication_denial; -use crate::credentials::models::{AuditContext, SessionAccess}; +use crate::credentials::models::{AuditContext, IssuedSession, SessionAccess}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -116,7 +119,7 @@ pub(crate) async fn authenticate( }; let mut context = context; context.actor = format!("principal:{}", identity.principal_id); - context.granting_role = "password-session".into(); + context.granting_role = "browser-session".into(); request.extensions_mut().insert(context); request.extensions_mut().insert(identity); Ok(next.run(request).await) @@ -158,6 +161,36 @@ pub(crate) fn browser_cookie<'a>(headers: &'a HeaderMap, name: &str) -> Option<& session } +pub(crate) fn external_session(session: IssuedSession) -> Result { + let response = Json(ExternalIdentityResponse { + principal_id: session.identity.principal_id, + session_id: session.identity.session_id, + expires_at: session.identity.expires_at, + csrf_token: session.csrf.clone(), + }) + .into_response(); + session_cookies(response, &session) +} + +pub(crate) fn session_cookies( + mut response: Response, + session: &IssuedSession, +) -> Result { + set_cookie( + &mut response, + (PASSWORD_COOKIE, &session.bearer), + PASSWORD_SESSION_SECONDS, + (true, "Strict"), + )?; + set_cookie( + &mut response, + (PASSWORD_CSRF_COOKIE, &session.csrf), + PASSWORD_SESSION_SECONDS, + (false, "Strict"), + )?; + Ok(response) +} + pub(crate) fn set_cookie( response: &mut Response, (name, value): (&str, &str), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs index 916483eed8..a1fcfc2c99 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs @@ -2,8 +2,9 @@ use std::sync::Arc; +use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::password_transport::{ - deny, guard, parse_origin, single_header, + deny, external_session, guard, parse_origin, single_header, }; use crate::api::constants::PROXY_USER_HEADER; use crate::api::errors::ApiError; @@ -11,14 +12,15 @@ use crate::api::models::{ AuthenticationDenial, ExternalIdentityResponse, TrustedProxyAuthentication, }; use crate::credentials::main::authenticate_bound_person::authenticate_bound_person; -use crate::credentials::models::{AuditContext, ExternalIdentityMechanism}; +use crate::credentials::main::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::models::{AuditContext, AuthenticationAttempt, ExternalIdentityMechanism}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; use axum::extract::State; -use axum::http::{HeaderMap, HeaderValue}; -use axum::response::{IntoResponse, Response}; +use axum::http::{HeaderMap, HeaderValue, StatusCode}; +use axum::response::Response; use axum::routing::get; -use axum::{Extension, Json, Router, middleware}; +use axum::{Extension, Router, middleware}; pub(crate) fn router( store: &ControlStore, @@ -52,15 +54,16 @@ pub(crate) fn router( .collect(); Ok(Router::new() .route("/api/auth/proxy", get(identity)) + .merge(password_sessions::protected(store)) .layer(middleware::from_fn_with_state(store.clone(), guard)) .layer(Extension(origin)) .layer(Extension(Arc::new(settings)))) } #[utoipa::path(get, path = "/api/auth/proxy", operation_id = "proxyIdentity", - description = "Requires a configured socket peer and an upstream-replaced X-SQB-User subject. Does not issue sessions or grant permissions.", + description = "Requires a configured socket peer and upstream-replaced X-SQB-User subject; issues a bounded browser session without provisioning, personal API tokens or implicit grants.", security(("proxy" = [])), - responses((status = 200, body = ExternalIdentityResponse), (status = 401, body = AuthenticationDenial)))] + responses((status = 200, body = ExternalIdentityResponse), (status = 401, body = AuthenticationDenial), (status = 429, body = AuthenticationDenial)))] pub(crate) async fn identity( State(store): State, Extension(settings): Extension>, @@ -79,16 +82,23 @@ pub(crate) async fn identity( let Some(principal_id) = principal else { return deny(&store, &context, "unbound_proxy_subject").await; }; - if authenticate_bound_person( + let (_, allowed) = + reserve_authentication_attempt(&store, &context, AuthenticationAttempt::IdentityProvider) + .await?; + if !allowed { + let mut response = deny(&store, &context, "proxy_login_throttled").await?; + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + return Ok(response); + } + let Some(session) = authenticate_bound_person( &store, &context, principal_id, ExternalIdentityMechanism::TrustedProxy, ) .await? - .is_none() - { + else { return deny(&store, &context, "inactive_proxy_person").await; - } - Ok(Json(ExternalIdentityResponse { principal_id }).into_response()) + }; + external_session(session) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs index 320f74ca76..0fcc2fae9a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -10,7 +10,9 @@ use crate::api::errors::ApiError; use crate::api::models::MutationResponse; use crate::authorization::main::assign_builtin_role::assign_builtin_role; use crate::authorization::main::change_administrator::change_administrator; -use crate::credentials::models::{AdministratorChange, AuditContext, RoleAssignmentRequest}; +use crate::credentials::models::{ + AdministratorAuthentication, AdministratorChange, AuditContext, RoleAssignmentRequest, +}; use crate::store::models::ControlStore; #[utoipa::path(post, path = "/api/admin/people/{principal_id}", operation_id = "changeAdministrator", @@ -19,7 +21,11 @@ use crate::store::models::ControlStore; responses((status = 200, body = MutationResponse), (status = 401, body = crate::api::models::AuthenticationDenial), (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] pub(crate) async fn change( - (State(store), Extension(context)): (State, Extension), + (State(store), Extension(context), Extension(authentication)): ( + State, + Extension, + Extension, + ), Path(principal_id): Path, headers: HeaderMap, Json(change): Json, @@ -29,7 +35,7 @@ pub(crate) async fn change( &store, &context, session_credentials(&headers), - (principal_id, change), + (principal_id, change, &authentication), ) .await?, })) @@ -41,11 +47,17 @@ pub(crate) async fn change( (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] pub(crate) async fn assign( (State(store), Extension(context)): (State, Extension), + Extension(authentication): Extension, headers: HeaderMap, Json(request): Json, ) -> Result, ApiError> { Ok(Json(MutationResponse { - applied: assign_builtin_role(&store, (&context, session_credentials(&headers)), &request) - .await?, + applied: assign_builtin_role( + &store, + (&context, session_credentials(&headers)), + &request, + &authentication, + ) + .await?, })) } diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c532994002..69ebcebf67 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -164,9 +164,23 @@ pub(crate) struct OidcCallback { pub(crate) code: Option, } -#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[derive(Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct ExternalIdentityResponse { pub principal_id: uuid::Uuid, + pub session_id: uuid::Uuid, + pub expires_at: chrono::DateTime, + pub csrf_token: String, +} + +impl std::fmt::Debug for ExternalIdentityResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ExternalIdentityResponse") + .field("principal_id", &self.principal_id) + .field("session_id", &self.session_id) + .field("expires_at", &self.expires_at) + .finish_non_exhaustive() + } } #[derive(Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs index 799b864352..4ec849bd1f 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs @@ -7,7 +7,7 @@ use crate::authorization::_helpers::administrators::{ deny_change, grant_installation_admin, remaining_administrator, }; use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{AdministratorChange, AuditContext}; +use crate::credentials::models::{AdministratorAuthentication, AdministratorChange, AuditContext}; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; use crate::store::models::{Change, ControlTransaction, Parameter}; @@ -16,6 +16,7 @@ pub(crate) async fn apply_change( transaction: ControlTransaction, context: &AuditContext, (principal_id, change): (Uuid, AdministratorChange), + authentication: &AdministratorAuthentication, ) -> Result { let (mut transaction, rows) = query(transaction, "SELECT p.disabled_at,a.assignment_id FROM sqb_principals p LEFT JOIN sqb_role_assignments a \ @@ -41,7 +42,8 @@ pub(crate) async fn apply_change( ) { let other; - (transaction, other) = remaining_administrator(transaction, principal_id).await?; + (transaction, other) = + remaining_administrator(transaction, principal_id, authentication).await?; if !other { return deny_change( transaction, diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs index 4e4e395a2b..2c8140ca9a 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs @@ -7,7 +7,8 @@ use crate::authorization::_helpers::administrator_changes::apply_change; use crate::authorization::main::deny_authorization::deny; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::{ - AdministratorChange, AuditContext, BuiltinRole, RoleAssignmentRequest, + AdministratorAuthentication, AdministratorChange, AuditContext, BuiltinRole, + RoleAssignmentRequest, }; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; @@ -17,6 +18,7 @@ pub(crate) async fn assign_role_in( transaction: ControlTransaction, context: &AuditContext, request: &RoleAssignmentRequest, + authentication: &AdministratorAuthentication, ) -> Result { if request .target @@ -49,6 +51,7 @@ pub(crate) async fn assign_role_in( AdministratorChange::Grant }, ), + authentication, ) .await; } diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs index 95db59c244..a1307e181e 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::authorization::main::deny_authorization::deny; use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::AuditContext; +use crate::credentials::models::{AdministratorAuthentication, AuditContext}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{Change, ControlTransaction, Parameter}; @@ -53,14 +53,28 @@ pub(crate) async fn grant_installation_admin( pub(crate) async fn remaining_administrator( transaction: ControlTransaction, principal_id: Uuid, + authentication: &AdministratorAuthentication, ) -> Result<(ControlTransaction, bool), ControlError> { let (transaction, rows) = query(transaction, - "SELECT a.principal_id FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) \ - JOIN sqb_password_credentials c ON c.principal_id=p.principal_id \ + "SELECT a.principal_id,c.credential_id FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) \ + LEFT JOIN sqb_password_credentials c ON c.principal_id=p.principal_id \ WHERE a.role_key='admin' AND a.location_id IS NULL AND a.target IS NULL AND a.revoked_at IS NULL \ - AND p.kind='person' AND p.disabled_at IS NULL AND p.principal_id<>$1 LIMIT 1", + AND p.kind='person' AND p.disabled_at IS NULL AND p.principal_id<>$1", vec![Parameter::Id(principal_id)]).await?; - Ok((transaction, !rows.is_empty())) + for person in rows { + let usable = match authentication { + AdministratorAuthentication::Password => !person["credential_id"].is_null(), + AdministratorAuthentication::BoundPeople(people) => { + let id: Uuid = serde_json::from_value(person["principal_id"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?; + people.contains(&id) + } + }; + if usable { + return Ok((transaction, true)); + } + } + Ok((transaction, false)) } pub(crate) async fn deny_change( diff --git a/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs index 7f2c3bb62c..81aaa63d4b 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs @@ -3,7 +3,8 @@ use crate::authorization::_helpers::administrator_roles::assign_role_in; use crate::authorization::main::authorize_session::authorize_session; use crate::credentials::models::{ - Action, AuditContext, DefaultRole, PermissionScope, RoleAssignmentRequest, + Action, AdministratorAuthentication, AuditContext, DefaultRole, PermissionScope, + RoleAssignmentRequest, }; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -13,6 +14,7 @@ pub async fn assign_builtin_role( store: &ControlStore, (context, credentials): (&AuditContext, (&str, &str)), request: &RoleAssignmentRequest, + authentication: &AdministratorAuthentication, ) -> Result { let (transaction, context) = authorize_session( begin(store, TransactionMode::Write).await?, @@ -26,5 +28,5 @@ pub async fn assign_builtin_role( DefaultRole::None, ) .await?; - assign_role_in(transaction, &context, request).await + assign_role_in(transaction, &context, request, authentication).await } diff --git a/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs index d07be01fbe..c5e2aeb59b 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs @@ -5,7 +5,8 @@ use uuid::Uuid; use crate::authorization::_helpers::administrator_changes::apply_change; use crate::authorization::main::authorize_session::authorize_session; use crate::credentials::models::{ - Action, AdministratorChange, AuditContext, DefaultRole, PermissionScope, + Action, AdministratorAuthentication, AdministratorChange, AuditContext, DefaultRole, + PermissionScope, }; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -15,7 +16,11 @@ pub async fn change_administrator( store: &ControlStore, context: &AuditContext, (bearer, csrf): (&str, &str), - (principal_id, change): (Uuid, AdministratorChange), + (principal_id, change, authentication): ( + Uuid, + AdministratorChange, + &AdministratorAuthentication, + ), ) -> Result { let (transaction, context) = authorize_session( begin(store, TransactionMode::Write).await?, @@ -32,5 +37,11 @@ pub async fn change_administrator( DefaultRole::None, ) .await?; - apply_change(transaction, &context, (principal_id, change)).await + apply_change( + transaction, + &context, + (principal_id, change), + authentication, + ) + .await } diff --git a/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs index 4ed5b5ea12..50b5f64436 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::authorization::_helpers::administrators::grant_installation_admin; use crate::authorization::main::deny_authorization::deny; use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::revoke_session::revoke_session; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; @@ -51,7 +52,6 @@ pub async fn recover_administrator( ) .await; }; - let now = transaction.now; let (transaction, _) = query( transaction, "UPDATE sqb_principals SET disabled_at=NULL WHERE principal_id=$1", @@ -60,12 +60,11 @@ pub async fn recover_administrator( .await?; let (transaction, _) = grant_installation_admin(transaction, context, principal_id, "offline_recovery").await?; - let (mut transaction, sessions) = query(transaction, - "UPDATE sqb_password_sessions SET revoked_at=$1 WHERE principal_id=$2 AND revoked_at IS NULL RETURNING session_id", - vec![Parameter::Time(now),Parameter::Id(principal_id)]).await?; + let (mut transaction, sessions_revoked) = + revoke_session(transaction, context, (principal_id, None)).await?; transaction = append_audit(transaction, context, (Uuid::now_v7(), "admin.offline_recovered"), (json!({"principal_id":principal_id}),json!({"enabled":person["disabled_at"].is_null()}), - json!({"enabled":true,"role":"admin","reason":reason,"sessions_revoked":sessions.len()}))).await?; + json!({"enabled":true,"role":"admin","reason":reason,"sessions_revoked":sessions_revoked}))).await?; transaction.changes.insert(Change::Credentials); commit(transaction).await } diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs new file mode 100644 index 0000000000..1de5b30f42 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs @@ -0,0 +1,67 @@ +//! Issue the same bounded, hashed browser credentials after the caller verifies a person. + +use chrono::{Duration, SubsecRound}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::constants::PASSWORD_SESSION_SECONDS; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, IssuedSession, SessionIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn issue_session( + transaction: ControlTransaction, + context: &AuditContext, + (principal_id, credential_id): (Uuid, Option), +) -> Result<(ControlTransaction, IssuedSession), ControlError> { + let now = transaction.now; + let session_id = Uuid::now_v7(); + let bearer = new_bearer(session_id)?; + let csrf = new_bearer(session_id)?; + let identity = SessionIdentity { + session_id, + principal_id, + expires_at: (now + Duration::seconds(PASSWORD_SESSION_SECONDS)).trunc_subsecs(6), + }; + let mut parameters = vec![ + Parameter::Id(session_id), + Parameter::Id(principal_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + Parameter::Text(sha256_hex(csrf.as_bytes())), + Parameter::Time(now), + Parameter::Time(identity.expires_at), + ]; + let (table, column, value) = match credential_id { + Some(id) => { + parameters.push(Parameter::Id(id)); + ("sqb_password_sessions", ",credential_id", ",$7") + } + None => ("sqb_external_sessions", "", ""), + }; + let (transaction, _) = query(transaction, + &format!("INSERT INTO {table} (session_id,principal_id,secret_hash,csrf_hash,created_at,expires_at{column}) VALUES ($1,$2,$3,$4,$5,$6{value})"), + parameters).await?; + let transaction = append_audit( + transaction, + context, + (session_id, "session.created"), + ( + json!({"principal_id": principal_id}), + json!(null), + json!(&identity), + ), + ) + .await?; + Ok(( + transaction, + IssuedSession { + identity, + bearer, + csrf, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index b6363dfde5..d527e17e44 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod browser_sessions; pub(crate) mod locations; pub(crate) mod password_sessions; pub(crate) mod passwords; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs index 7acd5d5a5c..d1709c1a72 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs @@ -1,14 +1,7 @@ //! Session creation rechecks the verified password version under the control-store write lock. -use chrono::{Duration, SubsecRound}; -use serde_json::json; -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use uuid::Uuid; - -use crate::credentials::_helpers::secrets::new_bearer; -use crate::credentials::constants::PASSWORD_SESSION_SECONDS; -use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{AuditContext, IssuedSession, SessionIdentity, VerifiedPassword}; +use crate::credentials::_helpers::browser_sessions; +use crate::credentials::models::{AuditContext, IssuedSession, VerifiedPassword}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -19,7 +12,6 @@ pub(crate) async fn issue_session( proof: VerifiedPassword, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Write).await?; - let now = transaction.now; let (transaction, rows) = query(transaction, "SELECT p.principal_id FROM sqb_principals p JOIN sqb_password_credentials c ON p.principal_id=c.principal_id \ WHERE p.principal_id=$1 AND c.credential_id=$2 AND p.kind='person' AND p.disabled_at IS NULL", @@ -28,42 +20,17 @@ pub(crate) async fn issue_session( commit(transaction).await?; return Ok(None); } - let session_id = Uuid::now_v7(); - let bearer = new_bearer(session_id)?; - let csrf = new_bearer(session_id)?; - let identity = SessionIdentity { - session_id, - principal_id: proof.principal_id, - expires_at: (now + Duration::seconds(PASSWORD_SESSION_SECONDS)).trunc_subsecs(6), - }; - let (transaction, _) = query(transaction, - "INSERT INTO sqb_password_sessions (session_id, principal_id, credential_id, secret_hash, csrf_hash, created_at, expires_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7)", - vec![Parameter::Id(session_id), Parameter::Id(proof.principal_id), Parameter::Id(proof.credential_id), - Parameter::Text(sha256_hex(bearer.as_bytes())), Parameter::Text(sha256_hex(csrf.as_bytes())), - Parameter::Time(now), Parameter::Time(identity.expires_at)]).await?; let context = AuditContext { actor: format!("principal:{}", proof.principal_id), granting_role: "password-session".into(), ..context.clone() }; - commit( - append_audit( - transaction, - &context, - (session_id, "session.created"), - ( - json!({"principal_id": proof.principal_id}), - json!(null), - json!(&identity), - ), - ) - .await?, + let (transaction, session) = browser_sessions::issue_session( + transaction, + &context, + (proof.principal_id, Some(proof.credential_id)), ) .await?; - Ok(Some(IssuedSession { - identity, - bearer, - csrf, - })) + commit(transaction).await?; + Ok(Some(session)) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs index c242dff68c..041c01a3ba 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs @@ -1,10 +1,11 @@ -//! Federation proves identity only; explicit bindings never provision accounts or grant roles. +//! Explicit federation bindings issue browser sessions without provisioning accounts or roles. use serde_json::json; use uuid::Uuid; +use crate::credentials::_helpers::browser_sessions::issue_session; use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{AuditContext, ExternalIdentityMechanism}; +use crate::credentials::models::{AuditContext, ExternalIdentityMechanism, IssuedSession}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -14,7 +15,7 @@ pub(crate) async fn authenticate_bound_person( context: &AuditContext, principal_id: Uuid, mechanism: ExternalIdentityMechanism, -) -> Result, ControlError> { +) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Write).await?; let (transaction, rows) = query(transaction, "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND kind='person' AND disabled_at IS NULL", @@ -32,6 +33,7 @@ pub(crate) async fn authenticate_bound_person( granting_role: granting_role.into(), ..context.clone() }; + let (transaction, session) = issue_session(transaction, &context, (principal_id, None)).await?; let identity = json!({"principal_id": principal_id}); commit( append_audit( @@ -43,5 +45,5 @@ pub(crate) async fn authenticate_bound_person( .await?, ) .await?; - Ok(Some(principal_id)) + Ok(Some(session)) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs index 67b8caa435..824bf7b382 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs @@ -1,4 +1,4 @@ -//! Live password versions invalidate all earlier sessions without per-process caches. +//! Live person eligibility, password versions and revocation apply to every browser request. use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; use crate::credentials::models::{SessionAccess, SessionIdentity, StoredSession}; @@ -16,9 +16,12 @@ pub(crate) async fn authenticate_session_in( }; let now = transaction.now; let (transaction, rows) = query(transaction, - "SELECT s.session_id, s.principal_id, s.expires_at, s.secret_hash, s.csrf_hash \ - FROM sqb_password_sessions s JOIN sqb_principals p ON p.principal_id=s.principal_id \ - JOIN sqb_password_credentials c ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ + "SELECT s.session_id, s.principal_id, s.expires_at, s.secret_hash, s.csrf_hash FROM (\ + SELECT s.session_id,s.principal_id,s.expires_at,s.secret_hash,s.csrf_hash,s.revoked_at \ + FROM sqb_password_sessions s JOIN sqb_password_credentials c \ + ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ + UNION ALL SELECT session_id,principal_id,expires_at,secret_hash,csrf_hash,revoked_at FROM sqb_external_sessions\ + ) s JOIN sqb_principals p ON p.principal_id=s.principal_id \ WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ AND p.kind='person' AND p.disabled_at IS NULL", vec![Parameter::Id(id), Parameter::Time(now)]).await?; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs index f7b5c6818b..bbac342acf 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs @@ -1,51 +1,56 @@ -//! Logout preserves session history and commits its security audit with revocation. +//! Revoke one session or every browser session without releasing the caller's write lock. use serde_json::json; use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; -use crate::credentials::models::{AuditContext, SessionIdentity}; +use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; -pub async fn revoke_session( - store: &ControlStore, +/// A missing session ID revokes all mechanisms; the caller audits its password/reset operation. +pub(crate) async fn revoke_session( + mut transaction: ControlTransaction, context: &AuditContext, - identity: &SessionIdentity, -) -> Result { - let transaction = begin(store, TransactionMode::Write).await?; + (principal_id, session_id): (Uuid, Option), +) -> Result<(ControlTransaction, usize), ControlError> { let now = transaction.now; - let (transaction, rows) = query( - transaction, - "UPDATE sqb_password_sessions SET revoked_at=$1 WHERE session_id=$2 \ - AND principal_id=$3 AND revoked_at IS NULL RETURNING session_id", - vec![ - Parameter::Time(now), - Parameter::Id(identity.session_id), - Parameter::Id(identity.principal_id), - ], - ) - .await?; - if rows.is_empty() { - commit(transaction).await?; - return Ok(false); + let mut parameters = vec![Parameter::Time(now), Parameter::Id(principal_id)]; + let predicate = match session_id { + Some(id) => { + parameters.push(Parameter::Id(id)); + "AND session_id=$3" + } + None => "", + }; + let mut revoked = 0; + for table in ["sqb_password_sessions", "sqb_external_sessions"] { + let rows; + (transaction, rows) = query( + transaction, + &format!("UPDATE {table} SET revoked_at=$1 WHERE principal_id=$2 {predicate} AND revoked_at IS NULL RETURNING session_id"), + parameters.clone(), + ) + .await?; + revoked += rows.len(); + } + if revoked == 0 { + return Ok((transaction, 0)); + } + if let Some(id) = session_id { + transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"revoked"), "session.revoked"), + ( + json!({"principal_id":principal_id}), + json!({"session_id":id,"revoked":false}), + json!({"session_id":id,"revoked":true}), + ), + ) + .await?; } - let mut transaction = append_audit( - transaction, - context, - ( - Uuid::new_v5(&identity.session_id, b"revoked"), - "session.revoked", - ), - ( - json!({"principal_id": identity.principal_id}), - json!({"session_id": identity.session_id, "revoked": false}), - json!({"session_id": identity.session_id, "revoked": true}), - ), - ) - .await?; transaction.changes.insert(Change::Credentials); - commit(transaction).await?; - Ok(true) + Ok((transaction, revoked)) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs index 60ac393916..9e68d961f7 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::credentials::_helpers::passwords::hash_password; use crate::credentials::constants::{MAX_PASSWORD_BYTES, MIN_PASSWORD_CHARACTERS}; use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::revoke_session::revoke_session; use crate::credentials::models::{AuditContext, PasswordAuthority}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; @@ -80,6 +81,7 @@ pub async fn set_password( vec![Parameter::Id(principal_id), Parameter::Id(credential_id), Parameter::Text(hash), Parameter::Time(now)], ) .await?; + let (transaction, _) = revoke_session(transaction, context, (principal_id, None)).await?; let mut transaction = append_audit( transaction, context, diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index b91d6f3945..05025f4def 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -97,6 +97,14 @@ pub enum DefaultRole { Viewer, } +/// Server-owned sign-in configuration, used to protect a genuinely usable administrator. +#[derive(Clone, Debug, Default)] +pub enum AdministratorAuthentication { + #[default] + Password, + BoundPeople(std::collections::BTreeSet), +} + /// Scope is computed by the server from the operation, never accepted as a claimed write set. #[derive(Clone, Debug, Serialize)] pub(crate) struct PermissionScope { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 4ec73f9146..9e561c1083 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -24,6 +24,7 @@ pub(crate) const AUTHENTICATION_REVISION: &str = include_str!("../authentication pub(crate) const OIDC_LOGIN_FLOWS: &str = include_str!("../oidc_login_flows.sql"); pub(crate) const ADMINISTRATOR_INITIALIZATION: &str = include_str!("../administrator_initialization.sql"); +pub(crate) const EXTERNAL_SESSIONS: &str = include_str!("../external_sessions.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/external_sessions.sql b/crates/sqlbuild-orchestration/src/store/external_sessions.sql new file mode 100644 index 0000000000..788fd7f9ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/external_sessions.sql @@ -0,0 +1,10 @@ +CREATE TABLE sqb_external_sessions ( + session_id {uuid} PRIMARY KEY, + principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time} +); +CREATE INDEX sqb_external_sessions_person ON sqb_external_sessions(principal_id); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 4fe30caac3..63dbb795db 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,9 +5,9 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, - LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, - PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, - WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, + EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, + OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, + SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -47,6 +47,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("authentication_revision", AUTHENTICATION_REVISION), ("oidc_login_flows", OIDC_LOGIN_FLOWS), ("administrator_initialization", ADMINISTRATOR_INITIALIZATION), + ("external_sessions", EXTERNAL_SESSIONS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 6b50cce76a..cb465fa31f 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -18,6 +18,8 @@ mod test_changes; mod test_contract; #[path = "test_development.rs"] mod test_development; +#[path = "test_external_sessions.rs"] +mod test_external_sessions; #[path = "test_location_health.rs"] mod test_location_health; #[path = "test_location_work.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs new file mode 100644 index 0000000000..22e3ccd8fd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -0,0 +1,379 @@ +//! Real HTTP federation sessions retain live roles, usable-admin protection and audited revocation. + +use std::collections::{BTreeMap, BTreeSet}; + +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, TrustedProxyAuthentication}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::ExternalSessionTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::{ + create_location::create_location, create_principal::create_principal, + set_password::set_password, verify_password::verify_password, +}; +use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_federated_people_when_administrators_and_sessions_change_then_live_authority_and_lifetime_hold() + { + let test_cases = [ + ExternalSessionTestCase { + description: "SQLite federated administrator/session journey", + backend: Backend::Sqlite, + expected_lifetime: 3600, + }, + ExternalSessionTestCase { + description: "PostgreSQL federated administrator/session journey", + backend: Backend::Postgres, + expected_lifetime: 3600, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json"); + let inputs: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop federation")) + .expect("inputs"); + let people = &inputs["administrator_journey"]; + let first_subject = people["bound"][0].as_str().expect("first bound person"); + let second_subject = people["bound"][1].as_str().expect("second bound person"); + let first = bootstrap_user(store, &context, first_subject) + .await + .expect("first admin"); + let second = create_principal(store, &context, (second_subject, PrincipalKind::Person)) + .await + .expect("second person"); + let unbound = create_principal( + store, + &context, + ( + people["unbound_password_person"] + .as_str() + .expect("unbound password person"), + PrincipalKind::Person, + ), + ) + .await + .expect("unbound person"); + set_password( + store, + &context, + PasswordAuthority::Operator(unbound), + "products-password-2026", + ) + .await + .expect("unbound password"); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("populated predecessor"); + for sql in [ + "DROP TABLE sqb_external_sessions", + "DELETE FROM sqb_schema_migrations WHERE name='external_sessions'", + ] { + (transaction, _) = query(transaction, sql, vec![]) + .await + .expect("reconstruct immediate predecessor"); + } + commit(transaction).await.expect("predecessor committed"); + migrate_store(&replica) + .await + .expect("additive session upgrade"); + migrate_store(store) + .await + .expect("idempotent session upgrade"); + assert!( + verify_password(store, unbound, "products-password-2026") + .await + .expect("preserved credential") + .is_some() + ); + let origin = inputs["origin"].as_str().expect("origin"); + let settings = TrustedProxyAuthentication { + origin: origin.into(), + trusted_peers: BTreeSet::from(["127.0.0.1".parse().expect("peer")]), + identities: BTreeMap::from([ + (first_subject.into(), first), + (second_subject.into(), second), + ]), + }; + let api = start_api_with_authentication( + store, + ApiAuthentication { + trusted_proxy: Some(settings.clone()), + ..Default::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + trusted_proxy: Some(settings), + ..Default::default() + }, + ) + .await; + let client = Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("client"); + let login = |url: &str, subject: &str| { + client + .get(format!("{url}/api/auth/proxy")) + .header(header::HOST, "control.example.com") + .header("x-sqb-user", subject) + }; + for condition in [ + "WHEN NEW.action='session.created'", + "WHEN NEW.action='proxy.authenticated'", + ] { + let fault = TransactionFault { + description: "failed federation publication", + table: "sqb_security_audit", + operation: "INSERT", + condition, + sqlite_body: "SELECT RAISE(ABORT, 'synthetic session audit failure')", + postgres_body: "RAISE EXCEPTION 'synthetic session audit failure'", + }; + install_fault(store, &fault).await; + let response = login(&api.url, first_subject) + .send() + .await + .expect("failed session publication"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + remove_fault(store, &fault).await; + } + let mut sessions = Vec::new(); + for subject in [first_subject, second_subject] { + let response = login(&api.url, subject).send().await.expect("SSO login"); + assert_eq!(response.status(), StatusCode::OK); + let cookie = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("session cookie") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let body = response.json::().await.expect("session response"); + sessions.push(( + cookie, + body["csrf_token"].as_str().expect("csrf").to_owned(), + body["session_id"] + .as_str() + .expect("id") + .parse::() + .expect("session UUID"), + )); + } + let request = |index: usize, path: &str| { + client + .post(format!("{}{path}", other.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, origin) + .header(header::COOKIE, &sessions[index].0) + .header("x-sqb-csrf", &sessions[index].1) + }; + let read = |index: usize, path: &str| { + client + .get(format!("{}{path}", other.url)) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &sessions[index].0) + }; + let roles = |principal_id| json!({"principal_id":principal_id,"role":"admin","location_id":null,"target":null}); + for id in [unbound, second] { + let response = request(0, "/api/admin/roles") + .json(&roles(id)) + .send() + .await + .expect("admin grant"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("applied grant")["applied"], + true + ); + } + // A bound person needs no local password; an unbound password account cannot replace it. + assert_eq!( + request(0, &format!("/api/admin/people/{first}")) + .json(&"revoke") + .send() + .await + .expect("SSO admin replacement") + .status(), + StatusCode::OK + ); + assert_eq!( + request(0, "/api/admin/roles") + .json(&roles(first)) + .send() + .await + .expect("live revoked admin") + .status(), + StatusCode::FORBIDDEN + ); + for operation in ["revoke", "disable"] { + let response = request(1, &format!("/api/admin/people/{second}")) + .json(&operation) + .send() + .await + .expect("final usable SSO admin"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let body = response.json::().await.expect("invariant denial"); + assert!(body["error"].as_str().expect("error").starts_with( + "cannot remove or disable the final usable installation administrator; decision " + )); + } + let location = create_location(store, "orders", &context) + .await + .expect("registered location"); + let metadata = format!("/api/control/locations/{location}/targets/development/runs"); + assert_eq!( + read(0, &metadata) + .send() + .await + .expect("no ambient role") + .status(), + StatusCode::FORBIDDEN + ); + let viewer = json!({"principal_id":first,"role":"viewer","location_id":location,"target":"development"}); + assert_eq!( + request(1, "/api/admin/roles") + .json(&viewer) + .send() + .await + .expect("viewer grant") + .status(), + StatusCode::OK + ); + assert_eq!( + read(0, &metadata) + .send() + .await + .expect("live viewer") + .status(), + StatusCode::OK + ); + let mut revocation = viewer; + revocation["revoke"] = json!(true); + assert_eq!( + request(1, "/api/admin/roles") + .json(&revocation) + .send() + .await + .expect("viewer revocation") + .status(), + StatusCode::OK + ); + assert_eq!( + read(0, &metadata) + .send() + .await + .expect("live viewer loss") + .status(), + StatusCode::FORBIDDEN + ); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("inspect sessions"), + "SELECT session_id,created_at,expires_at FROM sqb_external_sessions", + vec![], + ) + .await + .expect("committed sessions only"); + assert_eq!(rows.len(), 2, "failed audits did not create sessions"); + for row in rows { + let created: chrono::DateTime = + serde_json::from_value(row["created_at"].clone()).expect("created"); + let expires: chrono::DateTime = + serde_json::from_value(row["expires_at"].clone()).expect("expiry"); + // Session expiry is truncated to database microsecond precision. + assert!( + (test_case.expected_lifetime * 1_000_000 - 1 + ..=test_case.expected_lifetime * 1_000_000) + .contains( + &(expires - created) + .num_microseconds() + .expect("bounded lifetime") + ), + "{}", + test_case.description + ); + } + commit(transaction).await.expect("inspection committed"); + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("expire session"), + "UPDATE sqb_external_sessions SET expires_at=created_at WHERE session_id=$1", + vec![Parameter::Id(sessions[0].2)], + ) + .await + .expect("clock boundary"); + commit(transaction).await.expect("expiry committed"); + assert_eq!( + read(0, "/api/auth/session") + .send() + .await + .expect("expired session") + .status(), + StatusCode::UNAUTHORIZED + ); + set_password( + store, + &context, + PasswordAuthority::Operator(second), + "customers-password-2026", + ) + .await + .expect("owner credential rotation"); + assert_eq!( + read(1, "/api/auth/session") + .send() + .await + .expect("SSO session revoked on reset") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + login(&api.url, second_subject) + .send() + .await + .expect("fresh SSO after reset") + .status(), + StatusCode::OK + ); + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 08981f832a..7998bdb84c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -49,7 +49,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable for (version, location_cleanup) in [(3, ""), (2, "DROP TABLE sqb_location_instances;")] { execute(store, &format!( - "DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ + "DROP TABLE sqb_external_sessions; DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; \ diff --git a/crates/sqlbuild-orchestration/tests/api/test_oidc.rs b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs index 13b16da7bd..7b484e20fa 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_oidc.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs @@ -121,15 +121,62 @@ fn given_oidc_codes_when_verified_across_replicas_then_only_bound_live_people_au assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); assert!( - response.headers()[header::SET_COOKIE] - .to_str() - .expect("cleared cookie") - .contains("Max-Age=0") - ); - assert_eq!( - response.json::().await.expect("identity"), - json!({"principal_id":person}) + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| value + .to_str() + .expect("cookie") + .contains("__Host-sqb-oidc=;")) ); + let cookie = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("SSO session cookie") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let body = response + .json::() + .await + .expect("authenticated session"); + assert_eq!(body["principal_id"], json!(person)); + assert!(body.get("bearer").is_none()); + for url in [&api.url, &other.url] { + let session = provider + .client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &cookie) + .send() + .await + .expect("OIDC session shared across replicas"); + assert_eq!(session.status(), StatusCode::OK); + assert_eq!( + session.json::().await.expect("live identity")["principal_id"], + json!(person) + ); + assert_eq!( + provider + .client + .get(format!( + "{url}/api/control/locations/{}/targets/development/runs", + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &cookie) + .send() + .await + .expect("no implicit role") + .status(), + StatusCode::FORBIDDEN + ); + } assert_eq!( callback(&provider, &api.url, &flow).await.status(), StatusCode::UNAUTHORIZED @@ -221,6 +268,20 @@ fn given_oidc_codes_when_verified_across_replicas_then_only_bound_live_people_au .await .expect("live disable"); commit(transaction).await.expect("disable commit"); + for url in [&api.url, &other.url] { + assert_eq!( + provider + .client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &cookie) + .send() + .await + .expect("disabled person session") + .status(), + StatusCode::UNAUTHORIZED + ); + } assert_eq!( callback(&provider, &other.url, &flow).await.status(), StatusCode::UNAUTHORIZED @@ -255,11 +316,11 @@ fn given_oidc_codes_when_verified_across_replicas_then_only_bound_live_people_au assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); let transaction = begin(store, TransactionMode::Read).await.expect("evidence"); let (transaction, rows) = query(transaction, - "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_password_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='oidc.authenticated') AS authentications", + "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_external_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='oidc.authenticated') AS authentications", vec![]).await.expect("no provisioning"); assert_eq!( json!(rows[0]), - json!({"principals":2,"sessions":0,"tokens":0,"authentications":test_case.expected_authentications}), + json!({"principals":2,"sessions":test_case.expected_authentications,"tokens":0,"authentications":test_case.expected_authentications}), "{}", test_case.description ); diff --git a/crates/sqlbuild-orchestration/tests/api/test_proxy.rs b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs index fb04184725..5d3ca7a363 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_proxy.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs @@ -115,6 +115,7 @@ fn given_explicit_proxy_bindings_when_real_peers_and_live_people_are_checked_the .header(header::HOST, "control.example.com") .header(header::ORIGIN, value("origin")) }; + let mut sessions = Vec::new(); for url in [&api.url, &other.url] { let response = request(url) .header("x-sqb-user", value("subject")) @@ -123,10 +124,89 @@ fn given_explicit_proxy_bindings_when_real_peers_and_live_people_are_checked_the .expect("proxy authentication"); assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); - assert!(!response.headers().contains_key(header::SET_COOKIE)); + let cookie = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("browser session cookie") + .to_owned(); + for attribute in ["Secure", "HttpOnly", "SameSite=Strict", "Path=/"] { + assert!(cookie.contains(attribute)); + } + assert!(!cookie.contains("Domain=")); + let body = response + .json::() + .await + .expect("authenticated person session"); + assert_eq!(body["principal_id"], json!(person)); + assert!(body.get("bearer").is_none()); + sessions.push(( + cookie.split(';').next().expect("cookie pair").to_owned(), + body["csrf_token"].as_str().expect("CSRF").to_owned(), + )); + } + for url in [&api.url, &other.url] { + let response = client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &sessions[0].0) + .send() + .await + .expect("shared browser session"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("session identity")["principal_id"], + json!(person) + ); + let denied = client + .get(format!( + "{url}/api/control/locations/{}/targets/development/runs", + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &sessions[0].0) + .send() + .await + .expect("no implicit viewer permission"); + assert_eq!(denied.status(), StatusCode::FORBIDDEN); + } + let logout = |csrf: &str| { + client + .post(format!("{}/api/auth/logout", other.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, value("origin")) + .header(header::COOKIE, &sessions[0].0) + .header("x-sqb-csrf", csrf) + }; + assert_eq!( + logout("wrong-csrf") + .send() + .await + .expect("CSRF rejection") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + logout(&sessions[0].1) + .send() + .await + .expect("audited logout") + .status(), + StatusCode::NO_CONTENT + ); + for url in [&api.url, &other.url] { assert_eq!( - response.json::().await.expect("identity only"), - json!({"principal_id": person}) + client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &sessions[0].0) + .send() + .await + .expect("revoked across replicas") + .status(), + StatusCode::UNAUTHORIZED ); } let mut denials = Vec::new(); @@ -221,6 +301,17 @@ fn given_explicit_proxy_bindings_when_real_peers_and_live_people_are_checked_the .expect("disabled denial") .decision_id, ); + assert_eq!( + client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &sessions[1].0) + .send() + .await + .expect("disabled session denied live") + .status(), + StatusCode::UNAUTHORIZED + ); } let mut transaction = begin(store, TransactionMode::Read) .await @@ -245,10 +336,10 @@ fn given_explicit_proxy_bindings_when_real_peers_and_live_people_are_checked_the assert_eq!(row["granting_role"], "trusted-proxy"); assert_eq!(row["ip_address"], "127.0.0.1"); } - let (transaction, rows) = query(transaction, "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_password_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens", vec![]).await.expect("no implicit provisioning"); + let (transaction, rows) = query(transaction, "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_external_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens", vec![]).await.expect("no implicit provisioning or personal tokens"); assert_eq!( json!(rows[0]), - json!({"principals": 2, "sessions": 0, "tokens": 0}) + json!({"principals": 2, "sessions": test_case.expected_authentications, "tokens": 0}) ); commit(transaction).await.expect("inspection complete"); stop_api(untrusted).await; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 90355b36c0..ee1b9619be 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -88,6 +88,12 @@ pub(crate) struct ProxyApiTestCase { pub(crate) expected_authentications: usize, } +pub(crate) struct ExternalSessionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_lifetime: i64, +} + pub(crate) struct OidcApiTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 8f6a746dc1..9d8561104f 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -23,6 +23,7 @@ pub(crate) async fn fingerprint_history( // Upgrade the immediate predecessor while published snapshots and freshness facts exist. let mut transaction = begin(store, TransactionMode::Write).await.expect("upgrade"); for table in [ + "sqb_external_sessions", "sqb_role_assignments", "sqb_admin_initialization", "sqb_oidc_login_flows", @@ -48,7 +49,7 @@ pub(crate) async fn fingerprint_history( .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows','administrator_initialization')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows','administrator_initialization','external_sessions')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs index 3aedf94789..54743a7b94 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs @@ -230,9 +230,15 @@ fn given_password_sessions_when_replicas_race_or_auditing_fails_then_throttles_a }; install_fault(store, &revocation_fault).await; assert!( - revoke_session(store, context, &customer_session.identity) - .await - .is_err() + revoke_session( + begin(store, TransactionMode::Write) + .await + .expect("logout transaction"), + context, + (customer, Some(customer_session.identity.session_id)) + ) + .await + .is_err() ); remove_fault(store, &revocation_fault).await; assert!( @@ -289,16 +295,35 @@ fn given_password_sessions_when_replicas_race_or_auditing_fails_then_throttles_a .await .is_err() ); - assert!( - revoke_session(store, context, &customer_session.identity) + let PasswordLogin::Authenticated(customer_session) = + login_password(store, context, ("customers-reader", replacement)) .await - .expect("logout") - ); - assert!( - !revoke_session(&replica, context, &customer_session.identity) + .expect("fresh login after password reset") + else { + panic!("expected fresh session for logout"); + }; + let (transaction, revoked) = revoke_session( + begin(store, TransactionMode::Write) .await - .expect("idempotent logout") - ); + .expect("logout transaction"), + context, + (customer, Some(customer_session.identity.session_id)), + ) + .await + .expect("logout"); + assert_eq!(revoked, 1); + commit(transaction).await.expect("publish logout"); + let (transaction, revoked) = revoke_session( + begin(&replica, TransactionMode::Write) + .await + .expect("retry logout transaction"), + context, + (customer, Some(customer_session.identity.session_id)), + ) + .await + .expect("idempotent logout"); + assert_eq!(revoked, 0); + commit(transaction).await.expect("finish retry"); let PasswordLogin::Authenticated(disabled_session) = login_password(store, context, ("customers-reader", replacement)) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs index 34d6000912..4790b6db06 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs @@ -12,7 +12,8 @@ use crate::credentials::main::{ revoke_session::revoke_session, set_password::set_password, }; use crate::credentials::models::{ - AdministratorChange, AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind, + AdministratorAuthentication, AdministratorChange, AuditContext, PasswordAuthority, + PasswordLogin, PrincipalKind, }; use crate::credentials::tests::helpers::authentication_revision; use crate::credentials::tests::test_types::AdministratorTestCase; @@ -106,7 +107,11 @@ fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_o store, &context, second_credentials, - (second, AdministratorChange::Grant) + ( + second, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .is_err(), @@ -117,15 +122,24 @@ fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_o store, &context, (first_credentials.0, "wrong-csrf"), - (second, AdministratorChange::Grant) + ( + second, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .is_err() ); assert!( - change_administrator(store, &context, first_credentials, (first, change)) - .await - .is_err(), + change_administrator( + store, + &context, + first_credentials, + (first, change, &AdministratorAuthentication::Password) + ) + .await + .is_err(), "cannot remove sole administrator" ); assert!( @@ -133,7 +147,11 @@ fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_o store, &context, first_credentials, - (second, AdministratorChange::Grant) + ( + second, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .expect("authenticated administrator grant") @@ -146,7 +164,11 @@ fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_o store, &context, first_credentials, - (second, AdministratorChange::Grant) + ( + second, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .expect("idempotent grant") @@ -158,8 +180,18 @@ fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_o revision ); let (left, right) = tokio::join!( - change_administrator(store, &context, first_credentials, (first, change)), - change_administrator(&replica, &context, second_credentials, (second, change)) + change_administrator( + store, + &context, + first_credentials, + (first, change, &AdministratorAuthentication::Password) + ), + change_administrator( + &replica, + &context, + second_credentials, + (second, change, &AdministratorAuthentication::Password) + ) ); assert_eq!( [&left, &right] @@ -186,15 +218,27 @@ fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_o assert_eq!(audit[0]["actor"], json!(format!("principal:{first}"))); assert_eq!(audit[0]["granting_role"], json!("admin")); commit(transaction).await.expect("finish audit read"); - revoke_session(store, &context, &sessions[survivor].identity) - .await - .expect("logout"); + let identity = &sessions[survivor].identity; + let (transaction, _) = revoke_session( + begin(store, TransactionMode::Write) + .await + .expect("logout transaction"), + &context, + (identity.principal_id, Some(identity.session_id)), + ) + .await + .expect("logout"); + commit(transaction).await.expect("publish logout"); assert!( change_administrator( &replica, &context, survivor_credentials, - (first, AdministratorChange::Grant) + ( + first, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .is_err(), @@ -318,7 +362,11 @@ fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_aut store, &context, credentials, - (second, AdministratorChange::Grant) + ( + second, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .is_err() @@ -335,7 +383,11 @@ fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_aut store, &context, credentials, - (second, AdministratorChange::Grant) + ( + second, + AdministratorChange::Grant, + &AdministratorAuthentication::Password + ), ) .await .expect("retry grant"), @@ -346,7 +398,11 @@ fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_aut store, &context, credentials, - (first, AdministratorChange::Disable) + ( + first, + AdministratorChange::Disable, + &AdministratorAuthentication::Password + ), ) .await .is_err(), @@ -366,7 +422,11 @@ fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_aut store, &context, credentials, - (first, AdministratorChange::Disable) + ( + first, + AdministratorChange::Disable, + &AdministratorAuthentication::Password + ), ) .await .is_err() @@ -377,7 +437,11 @@ fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_aut store, &context, credentials, - (first, AdministratorChange::Disable) + ( + first, + AdministratorChange::Disable, + &AdministratorAuthentication::Password + ), ) .await .expect("retry disabling person"), diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs index 357439362f..bfea637a46 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs @@ -9,11 +9,13 @@ use crate::authorization::main::{ bootstrap_user::bootstrap_user, recover_administrator::recover_administrator, }; use crate::credentials::main::{ + authenticate_bound_person::authenticate_bound_person, authenticate_session::authenticate_session, create_principal::create_principal, login_password::login_password, set_password::set_password, }; use crate::credentials::models::{ - AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind, SessionAccess, + AuditContext, ExternalIdentityMechanism, PasswordAuthority, PasswordLogin, PrincipalKind, + SessionAccess, }; use crate::credentials::tests::helpers::authentication_revision; use crate::credentials::tests::test_types::AdministratorRecoveryTestCase; @@ -102,6 +104,15 @@ fn given_lost_administrator_access_when_offline_recovery_runs_then_authority_and else { panic!("valid password"); }; + let external_session = authenticate_bound_person( + &fixture.store, + &context, + principal, + ExternalIdentityMechanism::Oidc, + ) + .await + .expect("verified OIDC identity") + .expect("SSO session"); assert!( begin_offline_recovery(&url).await.is_err(), "even idle normal clients exclude recovery" @@ -128,6 +139,7 @@ fn given_lost_administrator_access_when_offline_recovery_runs_then_authority_and ("sqb_security_audit", "INSERT"), ("sqb_role_assignments", "INSERT"), ("sqb_password_sessions", "UPDATE"), + ("sqb_external_sessions", "UPDATE"), ("sqb_authentication_revision", "UPDATE"), ] { let fault = TransactionFault { @@ -159,10 +171,10 @@ fn given_lost_administrator_access_when_offline_recovery_runs_then_authority_and revision ); let (transaction, rows) = query(begin(&fixture.store, TransactionMode::Read).await.expect("inspect rollback"), - "SELECT (SELECT COUNT(*) FROM sqb_principals WHERE disabled_at IS NOT NULL) AS disabled, (SELECT COUNT(*) FROM sqb_role_assignments WHERE revoked_at IS NULL) AS roles, (SELECT COUNT(*) FROM sqb_password_sessions WHERE revoked_at IS NULL) AS sessions, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.offline_recovered') AS recovered", vec![]).await.expect("rollback counts"); + "SELECT (SELECT COUNT(*) FROM sqb_principals WHERE disabled_at IS NOT NULL) AS disabled, (SELECT COUNT(*) FROM sqb_role_assignments WHERE revoked_at IS NULL) AS roles, (SELECT COUNT(*) FROM sqb_password_sessions WHERE revoked_at IS NULL) AS sessions, (SELECT COUNT(*) FROM sqb_external_sessions WHERE revoked_at IS NULL) AS external_sessions, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.offline_recovered') AS recovered", vec![]).await.expect("rollback counts"); assert_eq!( serde_json::to_value(&rows[0]).expect("row"), - json!({"disabled":1,"roles":0,"sessions":1,"recovered":0}) + json!({"disabled":1,"roles":0,"sessions":1,"external_sessions":1,"recovered":0}) ); commit(transaction).await.expect("finish rollback read"); } @@ -202,6 +214,16 @@ fn given_lost_administrator_access_when_offline_recovery_runs_then_authority_and .expect("old session") .is_none() ); + assert!( + authenticate_session( + &fixture.store, + &external_session.bearer, + SessionAccess::Read + ) + .await + .expect("old SSO session") + .is_none() + ); assert!(matches!( login_password(&fixture.store, &context, (name, password)) .await @@ -226,7 +248,7 @@ fn given_lost_administrator_access_when_offline_recovery_runs_then_authority_and assert_eq!(rows[0]["granting_role"], "control-store-owner"); let after = &rows[0]["after_json"]; assert_eq!(after["reason"], reason); - assert_eq!(after["sessions_revoked"], 1); + assert_eq!(after["sessions_revoked"], 2); commit(transaction).await.expect("finish audit read"); close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index c30f34ce3e..32859f25fd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -11,8 +11,8 @@ use crate::credentials::main::{ create_principal::create_principal, login_password::login_password, set_password::set_password, }; use crate::credentials::models::{ - AdministratorChange, AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, PasswordLogin, - PrincipalKind, RoleAssignmentRequest, + AdministratorAuthentication, AdministratorChange, AuditContext, BuiltinRole, DefaultRole, + PasswordAuthority, PasswordLogin, PrincipalKind, RoleAssignmentRequest, }; use crate::credentials::tests::location_helpers::create_location; use crate::runs::main::{ @@ -161,7 +161,8 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi &context, (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()) ), - &assignment + &assignment, + &AdministratorAuthentication::Password, ) .await .expect("authenticated assignment") @@ -364,7 +365,11 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi store, &context, administrator_credentials, - (person, AdministratorChange::Disable) + ( + person, + AdministratorChange::Disable, + &AdministratorAuthentication::Password + ), ) .await .expect("disable person") @@ -376,7 +381,8 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi &RoleAssignmentRequest { revoke: true, ..assignment - } + }, + &AdministratorAuthentication::Password, ) .await .expect("revoke a disabled person's grant before reenabling") @@ -386,7 +392,11 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi store, &context, administrator_credentials, - (person, AdministratorChange::Enable) + ( + person, + AdministratorChange::Enable, + &AdministratorAuthentication::Password + ), ) .await .expect("enable person") diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 6a6ce0a5fe..0e3f1ca3a4 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -426,6 +426,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r .await .expect("upgrade fixture"); for table in [ + "sqb_external_sessions", "sqb_role_assignments", "sqb_admin_initialization", "sqb_oidc_login_flows", diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index af3a779d58..9d51b234d4 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -243,7 +243,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( - "DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ + "DROP TABLE sqb_external_sessions; DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ DROP TABLE sqb_password_sessions; DROP TABLE sqb_password_login_attempts; \ DROP TABLE sqb_password_credentials; DROP TABLE sqb_development_locations; DROP TABLE sqb_warehouse_fingerprints; DROP TABLE sqb_warehouse_freshness; \ diff --git a/deploy/README.md b/deploy/README.md index 410ac19c79..a40d6cefbe 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -111,22 +111,30 @@ TOKEN_UUID`, store its one-time replacement bearer, and recreate the location. T expiry stay unchanged, and the old credential is revoked atomically. Service-account principal tokens are separate from location/run tokens. Person identities do not become machine bearers. -For opt-in OIDC identity verification, mount a server-owned JSON file with `origin`, `issuer`, +For OIDC sign-in, mount a server-owned JSON file with `origin`, `issuer`, `client_id`, and explicit `identities` subject-to-existing-person UUID bindings; append `--oidc-config /etc/sqlbuild/oidc.json` to the server command. Both origin and issuer require HTTPS. Terminate HTTPS at a trusted proxy, preserve the external Host, restrict backend access, -and register `/api/auth/oidc/callback` at the issuer. This verifies an explicitly bound enabled -person; it does not provision people, issue a browser session, or grant roles. +and register `/api/auth/oidc/callback` at the issuer. The callback verifies an explicitly bound, +enabled person and issues a one-hour browser session. Trusted-proxy sign-in through +`GET /api/auth/proxy` uses the same session protections after verifying the configured socket +peer and upstream-replaced subject. Neither mechanism provisions people or grants roles. +The bearer stays in a Secure, HttpOnly, host-only cookie; mutations require the returned CSRF +token and matching origin. Sessions work across replicas sharing the installation, with live +person and role checks. `POST /api/auth/logout` revokes the session across replicas; password +resets and offline administrator recovery revoke all of that person's browser sessions. The local control-store owner initializes the first installation administrator with `sqb admin create-user NAME`, then provisions its password separately with `sqb admin set-password PRINCIPAL_ID`. Initialization is available once only; losing or disabling administrators never reopens it. `create-principal` creates a role-free identity, not another -administrator. With password authentication enabled, authenticated installation administrators +administrator. With password, OIDC or trusted-proxy authentication, installation administrators use `POST /api/admin/people/{principal_id}` with a JSON action of `grant`, `revoke`, `disable`, -or `enable`, their session cookie and CSRF header. This cannot remove or disable the final enabled, -password-provisioned administrator; an unprovisioned person is not a usable alternative in this -password-only administration path. It never authorizes a machine bearer or grants non-admin defaults. +or `enable`, their session cookie and CSRF header. This cannot remove or disable the final usable +administrator. A password installation requires another enabled, password-provisioned admin; +an OIDC or proxy installation requires another enabled admin in its configured identity bindings. +A password alone does not make an unbound person a usable SSO administrator. These endpoints +never authorize a machine bearer or grant non-admin defaults. If administrator access is lost, stop every server and administration process connected to the installation, then run `sqb admin recover-user PRINCIPAL_ID --reason "restore administrator access"` @@ -145,7 +153,7 @@ and `revoke` (false for a grant). An administrator assignment must be installati Other built-ins can be installation-wide or restricted to a location or target. Revocation also works while a person is disabled, before they are enabled again. -Password sessions use `GET /api/control/locations/{location_id}/targets/{target}/runs` +Browser sessions use `GET /api/control/locations/{location_id}/targets/{target}/runs` for recent run metadata, `POST /api/control/runs` for published-job requests, and `POST /api/control/triggers` for trigger overrides. Use `POST /api/control/runs/cancel` with `run_id`, `event_id` and `reason` to request cancellation; use diff --git a/fensu.toml b/fensu.toml index 51d4478e3c..ff15538e20 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1573,8 +1573,8 @@ reason = "Issued credentials implement Debug manually to redact bearer and CSRF [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/api/models.rs" -symbols = ["PasswordLoginRequest", "PasswordChangeRequest", "PasswordSessionResponse"] -reason = "Password transport models implement Debug manually to redact passwords and CSRF secrets; other models derive Debug." +symbols = ["PasswordLoginRequest", "PasswordChangeRequest", "PasswordSessionResponse", "ExternalIdentityResponse"] +reason = "Browser transport models implement Debug manually to redact passwords and CSRF secrets; other models derive Debug." [[targets.rust.rule_exceptions]] rule = "FPRSS201" From aa6f2d07c4cd4917c6dcce6cc29c7d9a3ca613c2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 05:39:40 +0000 Subject: [PATCH 084/138] feat: reconcile bounded warehouse history mirrors --- .../warehouse_reconciliation.json | 47 +++ clients/typescript/src/schema.d.ts | 29 +- .../_helpers/resources/warehouse_metadata.rs | 25 +- .../src/platform/_helpers/work_client.rs | 36 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 4 +- .../src/store/warehouse_reconciliation.sql | 22 ++ .../_helpers/fingerprints.rs | 20 +- .../warehouse_metadata/_helpers/freshness.rs | 39 ++ .../warehouse_metadata/_helpers/ingestion.rs | 49 ++- .../src/warehouse_metadata/_helpers/mod.rs | 2 + .../_helpers/reconciliation.rs | 181 +++++++++ .../main/ingest_fingerprints.rs | 48 ++- .../main/ingest_source_freshness.rs | 78 ++-- .../src/warehouse_metadata/models.rs | 34 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_external_sessions.rs | 5 +- .../tests/api/test_location_work.rs | 5 +- .../tests/api/test_warehouse_metadata.rs | 8 + .../tests/api/warehouse_metadata/helpers.rs | 9 +- .../reconciliation/helpers.rs | 345 ++++++++++++++++++ .../tests/sensors/test_runs.rs | 5 +- .../tests/snapshots/helpers.rs | 5 +- .../tests/store/helpers.rs | 25 +- fensu.toml | 4 + .../compiled_project/publication/helpers.py | 12 + .../publication/test_warehouse_metadata.py | 134 +++++++ 27 files changed, 1062 insertions(+), 112 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_reconciliation.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json new file mode 100644 index 0000000000..150c2c5fe2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json @@ -0,0 +1,47 @@ +{ + "day": "2026-01-01", + "obsolete_count": 205, + "retained_count": 1000, + "source_freshness": { + "event_id": "reconciliation-keep", + "state_database": null, + "state_schema": "reconcile_orders", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders", + "run_id": "orders-build", + "strategy": "max_column", + "value_kind": "integer", + "data_version": "17", + "data_version_hash": "orders-version-17", + "observed_at": "2026-01-01T10:00:00Z" + }, + "fingerprints": { + "event_id": "reconciliation-keep", + "state_database": null, + "state_schema": "reconcile_orders", + "node_type": "model", + "node_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders", + "run_id": "orders-build", + "definition_hash": "orders-definition", + "version_hash": "orders-version", + "schema_fingerprint": "orders-schema", + "definition": "select order_id from orders", + "metadata_json": "{}", + "ts": "2026-01-01T10:00:00Z" + }, + "retained_ids": [ + "reconciliation-keep", + "reconciliation-late", + "reconciliation-final-new", + "reconciliation-other-database", + "reconciliation-other-schema", + "reconciliation-before-day", + "reconciliation-after-day", + "reconciliation-preview" + ] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index bed94a0536..b236af6d7e 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -633,6 +633,7 @@ export interface components { FingerprintBatch: { /** Format: int64 */ generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; records: components["schemas"]["WarehouseFingerprintRecord"][]; /** Format: uuid */ session_id: string; @@ -1163,6 +1164,7 @@ export interface components { SourceFreshnessBatch: { /** Format: int64 */ generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; records: components["schemas"]["WarehouseFreshnessRecord"][]; /** Format: uuid */ session_id: string; @@ -1288,6 +1290,29 @@ export interface components { target_schema?: string | null; value_kind: string; }; + WarehouseReconciliationCursor: { + /** Format: uuid */ + event_id: string; + /** Format: date-time */ + observed_at: string; + }; + /** @description A complete, ordered key-set scan of one physical state table and UTC observation day. */ + WarehouseReconciliationPage: { + after?: null | components["schemas"]["WarehouseReconciliationCursor"]; + complete: boolean; + /** Format: date */ + day: string; + /** Format: int32 */ + page: number; + /** Format: uuid */ + scan_id: string; + state_database?: string | null; + state_schema: string; + }; + WarehouseSyncResponse: { + applied: boolean; + next_reconciliation_after?: null | components["schemas"]["WarehouseReconciliationCursor"]; + }; }; responses: never; parameters: never; @@ -1941,7 +1966,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["MutationResponse"]; + "application/json": components["schemas"]["WarehouseSyncResponse"]; }; }; 400: { @@ -1989,7 +2014,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["MutationResponse"]; + "application/json": components["schemas"]["WarehouseSyncResponse"]; }; }; 400: { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs index fa0606bb77..0330ca8532 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs @@ -6,16 +6,17 @@ use uuid::Uuid; use crate::api::_helpers::authentication::bearer::require_location; use crate::api::errors::ApiError; -use crate::api::models::MutationResponse; use crate::credentials::models::TokenIdentity; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::ingest_fingerprints::ingest_fingerprints; use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; -use crate::warehouse_metadata::models::{FingerprintBatch, SourceFreshnessBatch}; +use crate::warehouse_metadata::models::{ + FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, +}; #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", operation_id = "report_source_freshness", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, - responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), security(("bearer" = [])))] pub(crate) async fn report( @@ -23,16 +24,16 @@ pub(crate) async fn report( Extension(identity): Extension, Path((location_id, target)): Path<(Uuid, String)>, Json(batch): Json, -) -> Result, ApiError> { +) -> Result, ApiError> { require_location(&identity, location_id)?; - Ok(Json(MutationResponse { - applied: ingest_source_freshness(&store, (location_id, &target), &batch).await?, - })) + Ok(Json( + ingest_source_freshness(&store, (location_id, &target), &batch).await?, + )) } #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/fingerprints", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = FingerprintBatch, - responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), security(("bearer" = [])))] pub(crate) async fn report_fingerprints( @@ -40,9 +41,9 @@ pub(crate) async fn report_fingerprints( Extension(identity): Extension, Path((location_id, target)): Path<(Uuid, String)>, Json(batch): Json, -) -> Result, ApiError> { +) -> Result, ApiError> { require_location(&identity, location_id)?; - Ok(Json(MutationResponse { - applied: ingest_fingerprints(&store, (location_id, &target), &batch).await?, - })) + Ok(Json( + ingest_fingerprints(&store, (location_id, &target), &batch).await?, + )) } diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 09b423fee7..3761245961 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -8,26 +8,17 @@ use crate::api::models::MutationResponse; use crate::platform::models::{ LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, }; -use crate::warehouse_metadata::models::{FingerprintBatch, SourceFreshnessBatch}; +use crate::warehouse_metadata::models::{ + FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, +}; impl LocationWorkClient { pub async fn sync_fingerprints( &self, target: &str, batch: &FingerprintBatch, - ) -> Result { - let endpoint = self.connection.endpoint(&[ - "api", - "locations", - &self.location_id.to_string(), - "targets", - target, - "metadata", - "fingerprints", - ])?; - let response: MutationResponse = - client_response(self.connection.client.post(endpoint).json(batch)).await?; - Ok(response.applied) + ) -> Result { + self.sync_metadata((target, "fingerprints"), batch).await } /// Metadata shares the location's scoped pool, not the scheduler's execution authority. @@ -35,7 +26,16 @@ impl LocationWorkClient { &self, target: &str, batch: &SourceFreshnessBatch, - ) -> Result { + ) -> Result { + self.sync_metadata((target, "source-freshness"), batch) + .await + } + + async fn sync_metadata( + &self, + (target, table): (&str, &str), + batch: &impl serde::Serialize, + ) -> Result { let endpoint = self.connection.endpoint(&[ "api", "locations", @@ -43,11 +43,9 @@ impl LocationWorkClient { "targets", target, "metadata", - "source-freshness", + table, ])?; - let response: MutationResponse = - client_response(self.connection.client.post(endpoint).json(batch)).await?; - Ok(response.applied) + client_response(self.connection.client.post(endpoint).json(batch)).await } pub async fn poll( diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 9e561c1083..62ffc10aad 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -25,6 +25,7 @@ pub(crate) const OIDC_LOGIN_FLOWS: &str = include_str!("../oidc_login_flows.sql" pub(crate) const ADMINISTRATOR_INITIALIZATION: &str = include_str!("../administrator_initialization.sql"); pub(crate) const EXTERNAL_SESSIONS: &str = include_str!("../external_sessions.sql"); +pub(crate) const WAREHOUSE_RECONCILIATION: &str = include_str!("../warehouse_reconciliation.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 63dbb795db..239267fbb4 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -7,7 +7,8 @@ use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, - SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, render, verify_migrations, + SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_RECONCILIATION, render, + verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -48,6 +49,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("oidc_login_flows", OIDC_LOGIN_FLOWS), ("administrator_initialization", ADMINISTRATOR_INITIALIZATION), ("external_sessions", EXTERNAL_SESSIONS), + ("warehouse_reconciliation", WAREHOUSE_RECONCILIATION), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_reconciliation.sql b/crates/sqlbuild-orchestration/src/store/warehouse_reconciliation.sql new file mode 100644 index 0000000000..a5163587fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_reconciliation.sql @@ -0,0 +1,22 @@ +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN state_database TEXT; +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN state_schema TEXT; +UPDATE sqb_warehouse_fingerprints SET state_database=record_json ->> 'state_database', state_schema=record_json ->> 'state_schema'; +CREATE INDEX sqb_fingerprints_reconciliation ON sqb_warehouse_fingerprints (location_id,target,state_database,state_schema,observed_at,event_id); +ALTER TABLE sqb_warehouse_freshness ADD COLUMN state_database TEXT; +ALTER TABLE sqb_warehouse_freshness ADD COLUMN state_schema TEXT; +UPDATE sqb_warehouse_freshness SET state_database=record_json ->> 'state_database', state_schema=record_json ->> 'state_schema'; +CREATE INDEX sqb_freshness_reconciliation ON sqb_warehouse_freshness (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_scan_pages ( + scan_id {uuid} NOT NULL, + page BIGINT NOT NULL, + scope_hash TEXT NOT NULL, + request_hash TEXT NOT NULL, + complete BOOLEAN NOT NULL, + received_at {time} NOT NULL, + PRIMARY KEY (scan_id,page) +); +CREATE TABLE sqb_warehouse_scan_keys ( + scan_id {uuid} NOT NULL, + event_id {uuid} NOT NULL, + PRIMARY KEY (scan_id,event_id) +); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs index 860d810f29..12d55b10c5 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs @@ -4,7 +4,7 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use crate::store::errors::ControlError; -use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; use crate::warehouse_metadata::models::{FingerprintBatch, WarehouseFingerprintRecord}; pub(crate) fn validate_batch(batch: &FingerprintBatch) -> Result<(), ControlError> { @@ -25,12 +25,18 @@ pub(crate) fn validate_batch(batch: &FingerprintBatch) -> Result<(), ControlErro .iter() .filter_map(|record| record.event_id.as_deref()) .any(|value| value.trim().is_empty()); - if batch.generation < 1 || batch.records.len() > MAX_HISTORY_RECORDS || invalid || invalid_id { - return Err(ControlError::Invalid(format!( - "fingerprint sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", - ))); - } - Ok(()) + let invalid_scope = batch.reconciliation.as_ref().is_some_and(|page| { + batch.records.iter().any(|record| { + record.state_database != page.state_database + || record.state_schema != page.state_schema + || record.ts.date_naive() != page.day + }) + }); + validate_fields( + ("fingerprint", batch.generation, batch.records.len()), + invalid || invalid_id, + invalid_scope, + ) } /// Hash fixed predecessor fields only when the warehouse has no persisted identity. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs new file mode 100644 index 0000000000..bcaef6a86c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs @@ -0,0 +1,39 @@ +//! Validate bounded freshness observations and their optional reconciliation scope. + +use crate::store::errors::ControlError; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +pub(crate) fn validate_batch(batch: &SourceFreshnessBatch) -> Result<(), ControlError> { + let invalid_identity = batch + .records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.source_name, + &record.run_id, + &record.strategy, + &record.value_kind, + &record.data_version_hash, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_source_id = batch + .records + .iter() + .filter_map(|record| record.event_id.as_deref()) + .any(|value| value.trim().is_empty()); + let invalid_scope = batch.reconciliation.as_ref().is_some_and(|page| { + batch.records.iter().any(|record| { + record.state_database != page.state_database + || record.state_schema != page.state_schema + || record.observed_at.date_naive() != page.day + }) + }); + validate_fields( + ("source freshness", batch.generation, batch.records.len()), + invalid_identity || invalid_source_id, + invalid_scope, + ) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index 4d0f200fea..44fdd434d7 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -4,8 +4,11 @@ use uuid::Uuid; use crate::platform::main::require_instance::require_instance; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, query::query}; -use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::warehouse_metadata::_helpers::reconciliation::finish_reconciliation; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{WarehouseReconciliationPage, WarehouseSyncResponse}; pub(crate) async fn begin_ingestion( store: &ControlStore, @@ -38,3 +41,45 @@ pub(crate) fn scoped_event_id( .map_err(|error| ControlError::Invalid(error.to_string()))?; Ok(Uuid::new_v5(&location_id, &identity)) } + +pub(crate) async fn commit_ingestion( + mut transaction: ControlTransaction, + (table, location_id, target): (&str, Uuid, &str), + (page, keys): (Option<&WarehouseReconciliationPage>, &[Uuid]), + applied: bool, +) -> Result { + let mut response = WarehouseSyncResponse { + applied: false, + next_reconciliation_after: None, + }; + if let Some(page) = page { + (transaction, response) = + finish_reconciliation(transaction, (table, location_id, target), page, keys).await?; + } + response.applied |= applied; + if response.applied { + transaction + .changes + .insert(Change::NodeStatus(location_id, target.into())); + } + commit(transaction).await?; + Ok(response) +} + +pub(crate) fn validate_fields( + (kind, generation, count): (&str, i64, usize), + invalid_identity: bool, + invalid_scope: bool, +) -> Result<(), ControlError> { + if generation < 1 || count > MAX_HISTORY_RECORDS || invalid_identity { + return Err(ControlError::Invalid(format!( + "{kind} sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities" + ))); + } + if invalid_scope { + return Err(ControlError::Invalid(format!( + "{kind} reconciliation records must belong to its physical namespace and UTC day" + ))); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs index c4dcbb340a..4c4e4c12b9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod fingerprints; +pub(crate) mod freshness; pub(crate) mod ingestion; +pub(crate) mod reconciliation; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs new file mode 100644 index 0000000000..986f9541b3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs @@ -0,0 +1,181 @@ +//! Immutable scan receipts authorize only bounded, namespace/day-scoped mirror reconciliation. + +use chrono::NaiveTime; +use serde_json::{Value, json}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{ + WarehouseReconciliationCursor, WarehouseReconciliationPage, WarehouseSyncResponse, +}; + +pub(crate) async fn begin_reconciliation( + mut transaction: ControlTransaction, + scope: (&str, Uuid, &str), + page: &WarehouseReconciliationPage, + mut batch: Value, +) -> Result { + if page.state_schema.trim().is_empty() + || page.day.succ_opt().is_none() + || (!page.complete && page.after.is_some()) + || page + .after + .as_ref() + .is_some_and(|after| after.observed_at.date_naive() != page.day) + { + return Err(ControlError::Invalid( + "invalid reconciliation namespace, day or continuation".into(), + )); + } + let scope_hash = sha256_hex( + json!(( + scope, + &batch["session_id"], + &batch["generation"], + &page.state_database, + &page.state_schema, + page.day + )) + .to_string() + .as_bytes(), + ); + batch["reconciliation"] + .as_object_mut() + .ok_or_else(|| ControlError::Invalid("missing reconciliation page".into()))? + .remove("after"); + let request_hash = sha256_hex(batch.to_string().as_bytes()); + let latest; + (transaction, latest) = query(transaction, + "SELECT page,scope_hash,complete FROM sqb_warehouse_scan_pages WHERE scan_id=$1 ORDER BY page DESC LIMIT 1", + vec![Parameter::Id(page.scan_id)]).await?; + if latest + .first() + .is_some_and(|row| row["scope_hash"] != scope_hash) + { + return Err(ControlError::Invalid( + "reconciliation scan belongs to another scope or reader generation".into(), + )); + } + let previous; + (transaction, previous) = query( + transaction, + "SELECT request_hash FROM sqb_warehouse_scan_pages WHERE scan_id=$1 AND page=$2", + vec![ + Parameter::Id(page.scan_id), + Parameter::Integer(i64::from(page.page)), + ], + ) + .await?; + if let Some(previous) = previous.first() { + if previous["request_hash"] != request_hash { + return Err(ControlError::Invalid( + "reconciliation page retry changed its content".into(), + )); + } + return Ok(transaction); + } + let expected = latest.first().map_or(Some(0), |row| { + row["page"].as_u64().and_then(|page| page.checked_add(1)) + }); + if expected != Some(u64::from(page.page)) + || page.after.is_some() + || latest + .first() + .is_some_and(|row| row["complete"].as_bool() == Some(true)) + { + return Err(ControlError::Invalid( + "reconciliation pages must be contiguous and cannot extend a completed scan".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_warehouse_scan_pages (scan_id,page,scope_hash,request_hash,complete,received_at) VALUES ($1,$2,$3,$4,$5,$6)", + vec![Parameter::Id(page.scan_id), Parameter::Integer(i64::from(page.page)), Parameter::Text(scope_hash), + Parameter::Text(request_hash), Parameter::Boolean(page.complete), Parameter::Time(now)]).await?; + Ok(transaction) +} + +/// Each request visits at most one mirror page, including retained and concurrently received rows. +pub(crate) async fn finish_reconciliation( + mut transaction: ControlTransaction, + (table, location_id, target): (&str, Uuid, &str), + page: &WarehouseReconciliationPage, + keys: &[Uuid], +) -> Result<(ControlTransaction, WarehouseSyncResponse), ControlError> { + for key in keys { + (transaction, _) = query(transaction, + "INSERT INTO sqb_warehouse_scan_keys (scan_id,event_id) VALUES ($1,$2) ON CONFLICT (scan_id,event_id) DO NOTHING", + vec![Parameter::Id(page.scan_id), Parameter::Id(*key)]).await?; + } + let mut response = WarehouseSyncResponse { + applied: false, + next_reconciliation_after: None, + }; + if !page.complete { + return Ok((transaction, response)); + } + let end = page + .day + .succ_opt() + .ok_or_else(|| ControlError::Invalid("invalid reconciliation day".into()))?; + let start = page.day.and_time(NaiveTime::MIN).and_utc(); + let end = end.and_time(NaiveTime::MIN).and_utc(); + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(page.state_schema.clone()), + Parameter::Time( + page.after + .as_ref() + .map_or(start, |cursor| cursor.observed_at), + ), + Parameter::Time(end), + ]; + let database = match &page.state_database { + Some(database) => { + parameters.push(Parameter::Text(database.clone())); + "state_database=$6" + } + None => "state_database IS NULL", + }; + let lower_bound = if let Some(cursor) = &page.after { + parameters.push(Parameter::Id(cursor.event_id)); + format!("(observed_at,event_id)>($4,${})", parameters.len()) + } else { + "observed_at>=$4".into() + }; + let candidates; + (transaction, candidates) = query(transaction, &format!( + "SELECT observed_at,event_id FROM {table} WHERE location_id=$1 AND target=$2 AND state_schema=$3 \ + AND {lower_bound} AND observed_at<$5 AND {database} ORDER BY observed_at,event_id LIMIT {MAX_HISTORY_RECORDS}"), parameters).await?; + if candidates.len() == MAX_HISTORY_RECORDS { + response.next_reconciliation_after = Some( + serde_json::from_value::(json!(candidates.last())) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + } + if candidates.is_empty() { + return Ok((transaction, response)); + } + let mut parameters = vec![Parameter::Id(page.scan_id)]; + for candidate in &candidates { + let cursor: WarehouseReconciliationCursor = serde_json::from_value(json!(candidate)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + parameters.push(Parameter::Id(cursor.event_id)); + } + let placeholders = (2..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let removed; + (transaction, removed) = query(transaction, &format!( + "DELETE FROM {table} WHERE event_id IN ({placeholders}) \ + AND received_at<(SELECT received_at FROM sqb_warehouse_scan_pages WHERE scan_id=$1 AND page=0) \ + AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_scan_keys k WHERE k.scan_id=$1 AND k.event_id={table}.event_id) RETURNING event_id"), parameters).await?; + response.applied = !removed.is_empty(); + Ok((transaction, response)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs index e923464cf0..fc7a2a0d1e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs @@ -5,19 +5,20 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::{ - commit::commit, compress::compress, query::query, store_source_text::store_source_text, -}; -use crate::store::models::{Change, ControlStore, Parameter}; +use crate::store::main::{compress::compress, query::query, store_source_text::store_source_text}; +use crate::store::models::{ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::fingerprints::{source_event_id, validate_batch}; -use crate::warehouse_metadata::_helpers::ingestion::{begin_ingestion, scoped_event_id}; -use crate::warehouse_metadata::models::FingerprintBatch; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, scoped_event_id, +}; +use crate::warehouse_metadata::_helpers::reconciliation::begin_reconciliation; +use crate::warehouse_metadata::models::{FingerprintBatch, WarehouseSyncResponse}; pub async fn ingest_fingerprints( store: &ControlStore, (location_id, target): (Uuid, &str), batch: &FingerprintBatch, -) -> Result { +) -> Result { validate_batch(batch)?; let mut transaction = begin_ingestion( store, @@ -25,7 +26,17 @@ pub async fn ingest_fingerprints( (batch.session_id, batch.generation), ) .await?; + if let Some(page) = &batch.reconciliation { + transaction = begin_reconciliation( + transaction, + ("sqb_warehouse_fingerprints", location_id, target), + page, + json!(batch), + ) + .await?; + } let mut applied = false; + let mut keys: Vec = Vec::new(); for record in &batch.records { let source_event_id = source_event_id(record); let event_id = scoped_event_id( @@ -37,6 +48,9 @@ pub async fn ingest_fingerprints( ), &source_event_id, )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } let existing; (transaction, existing) = query( transaction, @@ -63,18 +77,18 @@ pub async fn ingest_fingerprints( .await?; let inserted; (transaction, inserted) = query(transaction, - "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), - Parameter::Json(record_json), Parameter::Time(received_at)]).await?; + Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; applied |= !inserted.is_empty(); } - if applied { - transaction - .changes - .insert(Change::NodeStatus(location_id, target.into())); - } - commit(transaction).await?; - Ok(applied) + commit_ingestion( + transaction, + ("sqb_warehouse_fingerprints", location_id, target), + (batch.reconciliation.as_ref(), &keys), + applied, + ) + .await } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index cb39441bee..de7e023696 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -6,54 +6,38 @@ use sqlbuild_state::freshness::models::FreshnessEventIdentity; use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::commit::commit; use crate::store::main::query::query; -use crate::store::models::{Change, ControlStore, Parameter}; -use crate::warehouse_metadata::_helpers::ingestion::{begin_ingestion, scoped_event_id}; -use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; -use crate::warehouse_metadata::models::SourceFreshnessBatch; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::freshness::validate_batch; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, scoped_event_id, +}; +use crate::warehouse_metadata::_helpers::reconciliation::begin_reconciliation; +use crate::warehouse_metadata::models::{SourceFreshnessBatch, WarehouseSyncResponse}; pub async fn ingest_source_freshness( store: &ControlStore, (location_id, target): (Uuid, &str), batch: &SourceFreshnessBatch, -) -> Result { - let invalid_identity = batch - .records - .iter() - .flat_map(|record| { - [ - &record.state_schema, - &record.source_name, - &record.run_id, - &record.strategy, - &record.value_kind, - &record.data_version_hash, - ] - }) - .any(|value| value.trim().is_empty()); - let invalid_source_id = batch.records.iter().any(|record| { - record - .event_id - .as_ref() - .is_some_and(|value| value.trim().is_empty()) - }); - if batch.generation < 1 - || batch.records.len() > MAX_HISTORY_RECORDS - || invalid_identity - || invalid_source_id - { - return Err(ControlError::Invalid(format!( - "source freshness sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", - ))); - } +) -> Result { + validate_batch(batch)?; let mut transaction = begin_ingestion( store, (location_id, target), (batch.session_id, batch.generation), ) .await?; + if let Some(page) = &batch.reconciliation { + transaction = begin_reconciliation( + transaction, + ("sqb_warehouse_freshness", location_id, target), + page, + json!(batch), + ) + .await?; + } let mut applied = false; + let mut keys: Vec = Vec::new(); for record in &batch.records { let source_event_id = record.event_id.clone().unwrap_or_else(|| { legacy_event_id(&FreshnessEventIdentity { @@ -80,20 +64,24 @@ pub async fn ingest_source_freshness( ), &source_event_id, )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } let received_at = transaction.now; let inserted; (transaction, inserted) = query(transaction, - "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at,state_database,state_schema) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), - Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at)]).await?; + Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at), + Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; applied |= !inserted.is_empty(); } - if applied { - transaction - .changes - .insert(Change::NodeStatus(location_id, target.into())); - } - commit(transaction).await?; - Ok(applied) + commit_ingestion( + transaction, + ("sqb_warehouse_freshness", location_id, target), + (batch.reconciliation.as_ref(), &keys), + applied, + ) + .await } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index cb926f1a92..c112395089 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -1,6 +1,6 @@ //! Existing warehouse state rows, with their physical state-table provenance. -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -30,6 +30,8 @@ pub struct SourceFreshnessBatch { pub session_id: Uuid, pub generation: i64, pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, } /// Physical namespace of SQLBuild's source-freshness state table, never a source query. @@ -78,6 +80,36 @@ pub struct FingerprintBatch { pub session_id: Uuid, pub generation: i64, pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + +/// A complete, ordered key-set scan of one physical state table and UTC observation day. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseReconciliationPage { + pub scan_id: Uuid, + pub page: u32, + pub state_database: Option, + pub state_schema: String, + pub day: NaiveDate, + pub complete: bool, + /// Repeat the identical final page with this mirror cursor until the response has none. + pub after: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseReconciliationCursor { + pub observed_at: DateTime, + pub event_id: Uuid, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct WarehouseSyncResponse { + pub applied: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_reconciliation_after: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index cb465fa31f..359a2ce7e2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -40,5 +40,7 @@ mod test_warehouse_metadata; mod test_worker; #[path = "warehouse_metadata/helpers.rs"] mod warehouse_metadata_helpers; +#[path = "warehouse_metadata/reconciliation/helpers.rs"] +mod warehouse_reconciliation_helpers; #[path = "work/helpers.rs"] mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs index 22e3ccd8fd..9960f27274 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -22,7 +22,9 @@ use crate::store::models::{Parameter, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::replica_helpers::second_store; -use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_helpers::{ + close_store, close_test_store, open_test_store, reconstruct_warehouse_predecessor, run, +}; use crate::store::tests::store_test_types::Backend; #[test] @@ -86,6 +88,7 @@ fn given_federated_people_when_administrators_and_sessions_change_then_live_auth ) .await .expect("unbound password"); + reconstruct_warehouse_predecessor(store).await; let mut transaction = begin(store, TransactionMode::Write) .await .expect("populated predecessor"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 7998bdb84c..3ded48bc65 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -20,7 +20,9 @@ use crate::api::tests::test_types::LocationWorkTestCase; use crate::api::tests::work_helpers::execute; use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::helpers::snapshot; -use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_helpers::{ + close_test_store, open_test_store, reconstruct_warehouse_predecessor, run, +}; use crate::store::tests::store_test_types::Backend; #[test] @@ -48,6 +50,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable // Exercise both deployed predecessors, including checksums and repeated startup. for (version, location_cleanup) in [(3, ""), (2, "DROP TABLE sqb_location_instances;")] { + reconstruct_warehouse_predecessor(store).await; execute(store, &format!( "DROP TABLE sqb_external_sessions; DROP TABLE sqb_role_assignments; DROP TABLE sqb_admin_initialization; \ DROP TABLE sqb_oidc_login_flows; DROP TABLE sqb_authentication_revision; \ diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 1b2c5aa8a1..3079949a9e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -126,6 +126,7 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit .sync_source_freshness("development", &typed) .await .expect("native client sync") + .applied ); expect_topics(&mut changes, &["node_status:development".into()]).await; assert!( @@ -133,6 +134,7 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit .sync_source_freshness("development", &typed) .await .expect("native client replay") + .applied ); drop(changes); let mut equivalent = batch.clone(); @@ -314,6 +316,12 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit (&client, &url, &token.bearer), ) .await; + crate::api::tests::warehouse_reconciliation_helpers::reconcile_history( + store, + (&client, &url, &token.bearer), + (session_id, preview_session), + ) + .await; stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 9d8561104f..3d25864517 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -12,6 +12,7 @@ use crate::store::main::{ use crate::store::models::{ControlStore, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::reconstruct_warehouse_predecessor; use crate::warehouse_metadata::models::FingerprintBatch; pub(crate) async fn fingerprint_history( @@ -21,6 +22,7 @@ pub(crate) async fn fingerprint_history( (client, freshness_url, bearer): (&Client, &str, &str), ) { // Upgrade the immediate predecessor while published snapshots and freshness facts exist. + reconstruct_warehouse_predecessor(store).await; let mut transaction = begin(store, TransactionMode::Write).await.expect("upgrade"); for table in [ "sqb_external_sessions", @@ -99,7 +101,8 @@ pub(crate) async fn fingerprint_history( outbound .sync_fingerprints("development", &typed) .await - .expect("sync"), + .expect("sync") + .applied, expected ); } @@ -246,6 +249,7 @@ pub(crate) async fn fingerprint_history( .sync_fingerprints("development", &typed) .await .expect("resync") + .applied ); assert_eq!( mirrored_fingerprints(store).await, @@ -289,6 +293,7 @@ pub(crate) async fn fingerprint_history( .sync_fingerprints("development", &marker) .await .expect("review marker sync") + .applied ); let rows = mirrored_fingerprints(store).await; let marker = rows @@ -311,6 +316,7 @@ pub(crate) async fn fingerprint_history( .sync_fingerprints("development", &scoped) .await .expect("distinct physical scopes") + .applied ); let preview: FingerprintBatch = serde_json::from_value(json!({ "session_id": preview_session_id, "generation": 1, "records": [record], @@ -321,6 +327,7 @@ pub(crate) async fn fingerprint_history( .sync_fingerprints("preview", &preview) .await .expect("distinct target scope") + .applied ); let after = mirrored_fingerprints(store).await; assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs new file mode 100644 index 0000000000..8fb2aac2cd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs @@ -0,0 +1,345 @@ +//! HTTP acceptance for bounded reconciliation, retry receipts and partial-scan safety. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; + +pub(crate) async fn reconcile_history( + store: &ControlStore, + (client, freshness_url, bearer): (&Client, &str, &str), + (session_id, preview_session_id): (Uuid, Uuid), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json"); + let fixture: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop reconciliation")) + .expect("fixture"); + for (kind, endpoint, table, timestamp) in [ + ( + "source_freshness", + "source-freshness", + "sqb_warehouse_freshness", + "observed_at", + ), + ( + "fingerprints", + "fingerprints", + "sqb_warehouse_fingerprints", + "ts", + ), + ] { + let url = freshness_url.replace("source-freshness", endpoint); + let record = fixture[kind].clone(); + let mut legacy = record.clone(); + legacy["event_id"] = Value::Null; + legacy["run_id"] = json!("orders-legacy-build"); + legacy[timestamp] = json!("2025-12-31T20:00:00-04:00"); + let mut originals = vec![record.clone(), legacy.clone()]; + for index in 0..fixture["obsolete_count"].as_u64().expect("obsolete count") { + let mut obsolete = record.clone(); + obsolete["event_id"] = json!(format!("reconciliation-obsolete-{index}")); + originals.push(obsolete); + } + for (id, field, value) in [ + ( + "reconciliation-other-database", + "state_database", + "analytics", + ), + ( + "reconciliation-other-schema", + "state_schema", + "reconcile_inventory", + ), + ( + "reconciliation-before-day", + timestamp, + "2025-12-31T23:59:59.999999Z", + ), + ( + "reconciliation-after-day", + timestamp, + "2026-01-02T00:00:00Z", + ), + ] { + let mut control = record.clone(); + control["event_id"] = json!(id); + control[field] = json!(value); + originals.push(control); + } + for records in originals.chunks(100) { + post( + (client, &url, bearer), + &json!({"session_id":session_id,"generation":1,"records":records}), + StatusCode::OK, + ) + .await; + } + let mut preview = record.clone(); + preview["event_id"] = json!("reconciliation-preview"); + post( + (client, &url.replace("/development/", "/preview/"), bearer), + &json!({"session_id":preview_session_id,"generation":1,"records":[preview]}), + StatusCode::OK, + ) + .await; + let before = observations(store, table).await; + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + let mut page = json!({"session_id":session_id,"generation":1,"records":[record.clone(),legacy], + "reconciliation":{"scan_id":Uuid::now_v7(),"page":0,"state_database":null,"state_schema":"reconcile_orders", + "day":fixture["day"],"complete":false,"after":null}}); + assert_eq!( + post((client, &url, bearer), &page, StatusCode::OK).await, + json!({"applied":false}) + ); + assert_eq!( + observations(store, table).await, + before, + "partial scan cannot delete" + ); + for (field, value) in [ + ("page", json!(2)), + ("state_database", json!("analytics")), + ("day", json!("2026-01-02")), + ] { + let mut invalid = page.clone(); + invalid["records"] = json!([]); + invalid["reconciliation"][field] = value; + post((client, &url, bearer), &invalid, StatusCode::BAD_REQUEST).await; + } + let mut changed = page.clone(); + changed["records"] = json!([record.clone()]); + post((client, &url, bearer), &changed, StatusCode::BAD_REQUEST).await; + let mut fenced = page.clone(); + fenced["generation"] = json!(2); + post((client, &url, bearer), &fenced, StatusCode::CONFLICT).await; + fenced = page.clone(); + fenced["session_id"] = json!(Uuid::now_v7()); + post((client, &url, bearer), &fenced, StatusCode::BAD_REQUEST).await; + let mut late = record.clone(); + late["event_id"] = json!("reconciliation-late"); + post( + (client, &url, bearer), + &json!({"session_id":session_id,"generation":1,"records":[late]}), + StatusCode::OK, + ) + .await; + let before_final = observations(store, table).await; + let mut final_new = record.clone(); + final_new["event_id"] = json!("reconciliation-final-new"); + page["records"] = json!([final_new]); + page["reconciliation"]["page"] = json!(1); + page["reconciliation"]["complete"] = json!(true); + for (fault_table, operation) in [ + ("sqb_warehouse_scan_pages", "INSERT"), + ("sqb_warehouse_scan_keys", "INSERT"), + (table, "DELETE"), + ] { + let fault = TransactionFault { + description: "reconciliation rollback", + table: fault_table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected reconciliation failure')", + postgres_body: "RAISE EXCEPTION 'injected reconciliation failure'", + }; + install_fault(store, &fault).await; + post( + (client, &url, bearer), + &page, + StatusCode::INTERNAL_SERVER_ERROR, + ) + .await; + remove_fault(store, &fault).await; + assert_eq!( + observations(store, table).await, + before_final, + "receipt, import, keys and deletion roll back together" + ); + } + let mut remaining = before_final.len() + 1; + for index in 0..3 { + let reply = tokio::time::timeout( + std::time::Duration::from_secs(5), + post((client, &url, bearer), &page, StatusCode::OK), + ) + .await + .expect("bounded reconciliation page"); + let now = observations(store, table).await.len(); + assert!( + remaining - now <= 100, + "at most one bounded mirror page is deleted" + ); + remaining = now; + page["reconciliation"]["after"] = reply["next_reconciliation_after"].clone(); + assert_eq!(page["reconciliation"]["after"].is_null(), index == 2); + } + assert!( + page["reconciliation"]["after"].is_null(), + "all bounded pages completed" + ); + let after = observations(store, table).await; + let mut retained: Vec<_> = after + .iter() + .filter_map(|row| row["record_json"]["event_id"].as_str()) + .collect(); + retained.sort(); + let mut expected: Vec<_> = fixture["retained_ids"] + .as_array() + .expect("expected identities") + .iter() + .map(|id| id.as_str().expect("identity")) + .collect(); + expected.sort(); + assert_eq!( + retained, expected, + "other namespaces/days/targets and concurrent delivery survive" + ); + assert_eq!( + after.len(), + expected.len() + 1, + "the fixed legacy observation also survives" + ); + page["reconciliation"]["after"] = Value::Null; + assert_eq!( + post((client, &url, bearer), &page, StatusCode::OK).await, + json!({"applied":false}), + "lost final response is retry-safe" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + let empty = json!({"session_id":session_id,"generation":1,"records":[],"reconciliation":{ + "scan_id":Uuid::now_v7(),"page":0,"state_database":null,"state_schema":"reconcile_orders","day":fixture["day"],"complete":true,"after":null}}); + assert_eq!( + post((client, &url, bearer), &empty, StatusCode::OK).await, + json!({"applied":true}) + ); + assert_eq!( + observations(store, table).await.len(), + 5, + "complete empty day deletes only its mirror scope" + ); + for records in originals.chunks(100) { + post( + (client, &url, bearer), + &json!({"session_id":session_id,"generation":1,"records":records}), + StatusCode::OK, + ) + .await; + } + assert_eq!( + observations(store, table).await, + before, + "empty-copy resync preserves exact identities and observations" + ); + retained_history( + (store, table), + (client, &url, bearer), + (session_id, &fixture, &record, timestamp), + ) + .await; + } +} + +async fn retained_history( + (store, table): (&ControlStore, &str), + (client, url, bearer): (&Client, &str, &str), + (session_id, fixture, record, timestamp): (Uuid, &Value, &Value, &str), +) { + let count = fixture["retained_count"].as_u64().expect("retained count"); + let records: Vec<_> = (0..count) + .map(|index| { + let mut record = record.clone(); + record["event_id"] = json!(format!("retained-{index}")); + record["state_database"] = json!("analytics"); + record["state_schema"] = json!("reconcile_retained"); + record[timestamp] = json!(format!( + "2026-01-01T10:{:02}:{:02}Z", + index / 120, + (index / 2) % 60 + )); + record + }) + .collect(); + for records in records.chunks(100) { + post( + (client, url, bearer), + &json!({"session_id":session_id,"generation":1,"records":records}), + StatusCode::OK, + ) + .await; + } + let mut page = json!({"session_id":session_id,"generation":1,"records":[],"reconciliation":{ + "scan_id":Uuid::now_v7(),"page":0,"state_database":"analytics","state_schema":"reconcile_retained", + "day":fixture["day"],"complete":false,"after":null}}); + let pages = records.len().div_ceil(100); + for (index, records) in records.chunks(100).enumerate() { + page["records"] = json!(records); + page["reconciliation"]["page"] = json!(index); + page["reconciliation"]["complete"] = json!(index + 1 == pages); + let reply = post((client, url, bearer), &page, StatusCode::OK).await; + assert_eq!( + reply["applied"], false, + "retained observations never change" + ); + page["reconciliation"]["after"] = reply["next_reconciliation_after"].clone(); + } + assert!(page["reconciliation"]["after"].is_object()); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + for index in 1..=pages { + let previous = page["reconciliation"]["after"].clone(); + let reply = post((client, url, bearer), &page, StatusCode::OK).await; + assert_eq!(reply["applied"], false); + page["reconciliation"]["after"] = reply["next_reconciliation_after"].clone(); + assert_ne!(page["reconciliation"]["after"], previous, "cursor advances"); + assert_eq!(page["reconciliation"]["after"].is_null(), index == pages); + } + }) + .await + .expect("retained-heavy continuation has a strict total deadline"); + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read retained"), + &format!("SELECT source_event_id FROM {table} WHERE state_database='analytics' AND state_schema='reconcile_retained' ORDER BY source_event_id"), + vec![], + ).await.expect("retained identities"); + commit(transaction).await.expect("close retained read"); + let mut expected: Vec<_> = (0..count) + .map(|index| format!("retained-{index}")) + .collect(); + expected.sort(); + assert_eq!( + rows.iter() + .map(|row| row["source_event_id"].as_str().expect("identity")) + .collect::>(), + expected + ); +} + +async fn post( + (client, url, bearer): (&Client, &str, &str), + body: &Value, + expected: StatusCode, +) -> Value { + let response = client + .post(url) + .bearer_auth(bearer) + .json(body) + .send() + .await + .expect("metadata request"); + let status = response.status(); + let body: Value = response.json().await.expect("metadata response"); + assert_eq!(status, expected, "{body}"); + body +} + +async fn observations(store: &ControlStore, table: &str) -> Vec { + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + &format!("SELECT event_id,target,source_event_id,record_json FROM {table} WHERE state_schema IN ('reconcile_orders','reconcile_inventory') ORDER BY event_id"), vec![]).await.expect("mirror facts"); + commit(transaction).await.expect("close read"); + rows +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 0e3f1ca3a4..2224aea579 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -28,7 +28,9 @@ use crate::store::models::{Parameter, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::replica_helpers::second_store; -use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_helpers::{ + close_store, close_test_store, open_test_store, reconstruct_warehouse_predecessor, run, +}; use crate::store::tests::store_test_types::Backend; #[test] @@ -422,6 +424,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r ) .await; // Recreate the exact pre-change index/migration state with populated run history. + reconstruct_warehouse_predecessor(store).await; let mut transaction = begin(store, TransactionMode::Write) .await .expect("upgrade fixture"); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 9d51b234d4..bf728d526e 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -15,7 +15,9 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::snapshots::tests::test_types::SchemaRejectionCase; use crate::store::tests::replica_helpers::second_store; -use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_helpers::{ + close_store, close_test_store, open_test_store, reconstruct_warehouse_predecessor, +}; use crate::store::tests::store_test_types::Backend; fn snapshot(location_id: Uuid) -> SnapshotInput { @@ -240,6 +242,7 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { let good = publish_snapshot(&store, &input) .await .expect("publish good snapshot"); + reconstruct_warehouse_predecessor(&store).await; let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); connection .execute_batch( diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index ba5fd692cc..3ea401f5e1 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -1,7 +1,8 @@ //! Real-database setup shared by storage, scheduling, and queue contracts. use crate::store::main::open_store::open_store; -use crate::store::models::ControlStore; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; use uuid::Uuid; use crate::store::tests::store_test_types::{Backend, TestStore}; @@ -73,3 +74,25 @@ pub(crate) async fn close_store(store: ControlStore) { }; pool.close().await; } + +pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("predecessor"); + for sql in [ + "DROP TABLE sqb_warehouse_scan_keys", + "DROP TABLE sqb_warehouse_scan_pages", + "DROP INDEX sqb_fingerprints_reconciliation", + "DROP INDEX sqb_freshness_reconciliation", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN state_database", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN state_schema", + "ALTER TABLE sqb_warehouse_freshness DROP COLUMN state_database", + "ALTER TABLE sqb_warehouse_freshness DROP COLUMN state_schema", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_reconciliation'", + ] { + (transaction, _) = query(transaction, sql, vec![]) + .await + .expect("reconstruct predecessor"); + } + commit(transaction).await.expect("commit predecessor"); +} diff --git a/fensu.toml b/fensu.toml index ff15538e20..a1bd617846 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1657,6 +1657,10 @@ reason = "Resource asset projections intentionally mirror their public JSON sche paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs"] reason = "Command document projections intentionally mirror the public envelope while preserving command-specific summaries: seeds require a plan and loads report skipped_count." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs"] +reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." + [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/migrations/_helpers/old_name_views.py" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 2a4c03f0a5..413762d597 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -138,6 +138,18 @@ def warehouse_history_page( return json.loads(result.stdout) +def sync_warehouse_history(api: SnapshotApi, table: str, batch: dict[str, Any]) -> dict[str, Any]: + """Use the public location transport and return its bounded reconciliation continuation.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/development/metadata/{table}", + data=json.dumps(batch).encode(), + headers={"Authorization": f"Bearer {api.bearer}", "Content-Type": "application/json"}, + ) + with urlopen(request, timeout=5) as response: + result: dict[str, Any] = json.load(response) + return result + + def wait_for_generation( api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path ) -> dict[str, Any]: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 440f4f3bad..b318ee8bc7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -7,6 +7,8 @@ from datetime import UTC, datetime, timedelta, timezone from pathlib import Path from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 import pytest @@ -14,11 +16,20 @@ Fingerprint, SourceFreshnessRecord, SourceFreshnessRenderers, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, read_latest_fingerprints, read_latest_source_freshness, write_fingerprints, write_source_freshness_records, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CatalogConnectionTestCase, FingerprintHistoryTestCase, @@ -28,8 +39,11 @@ append_contended_fingerprints, append_retried_fingerprints, open_legacy_warehouse, + snapshot_api, + sync_warehouse_history, warehouse_history_page, ) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @pytest.mark.parametrize( @@ -327,6 +341,126 @@ def page(after: list[str | None] | None = None, observed_since: str | None = Non assert not (project / "warehouse.db").exists() +@pytest.mark.parametrize( + "test_case", + [ + CatalogConnectionTestCase("DuckDB retained key set", "duckdb", "database", "retained-run"), + CatalogConnectionTestCase("SQLite retained key set", "sqlite", "path", "retained-run"), + ], + ids=lambda case: case.description, +) +def test_given_external_warehouse_retention_when_a_complete_scan_arrives_then_mirror_deletions_and_empty_resync_match_source( + test_case: CatalogConnectionTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "warehouse-project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for table in ("_sqlbuild_source_freshness", "_sqlbuild_fingerprints"): + _ = adapter.execute(connection=connection, sql=f"INSERT INTO {table} SELECT * FROM {table}") + _ = adapter.execute( + connection=connection, + sql=f"UPDATE {table} SET run_id='retained-run' WHERE rowid=(SELECT MAX(rowid) FROM {table})", + ) + adapter.close(connection) + declarations: Path = tmp_path / "declarations" + write_files(project_dir=declarations, files=TYPE_PROOF_FILES) + compiled: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=declarations), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with snapshot_api(tmp_path / "control") as api: + _ = build_orchestration_snapshot( + compiled, + json.dumps(METADATA | {"location_id": api.location_id, "target": "development"}), + declarations, + ).publish(api.url, api.bearer) + session_id: str = str(uuid4()) + heartbeat: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + data=json.dumps( + { + "session_id": session_id, + "event_id": str(uuid4()), + "target": "development", + "health": {"kind": "heartbeat"}, + } + ).encode(), + headers={"Authorization": f"Bearer {api.bearer}", "Content-Type": "application/json"}, + ) + with urlopen(heartbeat, timeout=5) as response: + assert json.load(response) == {"applied": True} + for kind, endpoint, mirror in [ + ("source_freshness", "source-freshness", "sqb_warehouse_freshness"), + ("fingerprints", "fingerprints", "sqb_warehouse_fingerprints"), + ]: + initial: dict[str, Any] = warehouse_history_page(project, kind) + assert len(initial["records"]) == 2 and initial["next_after"] is None + batch: dict[str, Any] = { + "session_id": session_id, + "generation": 1, + "records": initial["records"], + } + assert sync_warehouse_history(api, endpoint, batch) == {"applied": True} + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with closing(sqlite3.connect(control)) as database: + before: list[tuple[str, str]] = database.execute( + f"SELECT event_id,record_json FROM {mirror} ORDER BY event_id" + ).fetchall() + assert len(before) == 2 + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute( + connection=connection, sql=f"DELETE FROM _sqlbuild_{kind} WHERE run_id='legacy-run'" + ) + adapter.close(connection) + remaining: dict[str, Any] = warehouse_history_page(project, kind) + assert [row["run_id"] for row in remaining["records"]] == [test_case.expected_run_id] + batch["records"] = remaining["records"] + batch["reconciliation"] = { + "scan_id": str(uuid4()), + "page": 0, + "state_database": None, + "state_schema": "main", + "day": "2026-01-01", + "complete": False, + "after": None, + } + assert sync_warehouse_history(api, endpoint, batch) == {"applied": False} + with closing(sqlite3.connect(control)) as database: + assert database.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone() == (2,) + batch["records"] = [] + batch["reconciliation"].update(page=1, complete=True) + assert sync_warehouse_history(api, endpoint, batch) == {"applied": True} + originals_by_run: dict[str, tuple[str, str]] = { + json.loads(row[1])["run_id"]: row for row in before + } + expected: list[tuple[str, str]] = [originals_by_run[test_case.expected_run_id]] + with closing(sqlite3.connect(control)) as database: + assert ( + database.execute(f"SELECT event_id,record_json FROM {mirror}").fetchall() + == expected + ) + _ = database.execute(f"DELETE FROM {mirror}") + database.commit() + assert sync_warehouse_history( + api, + endpoint, + {"session_id": session_id, "generation": 1, "records": remaining["records"]}, + ) == {"applied": True} + with closing(sqlite3.connect(control)) as database: + assert ( + database.execute(f"SELECT event_id,record_json FROM {mirror}").fetchall() + == expected + ) + assert warehouse_history_page(project, kind) == remaining, ( + "sync does not mutate warehouse state" + ) + assert not (project / ".sqlbuild/control.sqlite3").exists() + + @pytest.mark.parametrize( "test_case", [ From a4f4ddbee99e41fbf1c140dfc1a16f75063bcbe6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:37:10 +0000 Subject: [PATCH 085/138] feat: automatically synchronize warehouse history --- .../automatic_warehouse_history.json | 6 + clients/typescript/src/schema.d.ts | 4 +- .../src/native_commands/_helpers/execution.rs | 226 +++++++++++++++++- .../src/native_commands/constants.rs | 5 + .../main/_location_metadata.rs | 122 ++++++++++ .../native_commands/main/_location_sensor.rs | 52 ++-- .../src/native_commands/main/_run_location.rs | 108 +++++---- .../src/native_commands/main/mod.rs | 1 + .../src/native_commands/models.rs | 22 ++ .../sqlbuild-cli/src/native_commands/types.rs | 7 +- .../src/api/constants.rs | 2 +- crates/sqlbuild-orchestration/src/api/mod.rs | 2 +- .../_helpers/fingerprints.rs | 2 +- .../warehouse_metadata/_helpers/freshness.rs | 4 +- .../_helpers/reconciliation.rs | 60 ++--- .../src/warehouse_metadata/models.rs | 6 +- .../reconciliation/helpers.rs | 34 +++ .../_helpers/location_command/command.rs | 25 +- .../native_host/warehouse_metadata.rs | 16 +- .../publication/_test_types.py | 14 ++ .../publication/test_warehouse_metadata.py | 145 +++++++++++ 21 files changed, 719 insertions(+), 144 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json new file mode 100644 index 0000000000..65c17a09a6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json @@ -0,0 +1,6 @@ +{ + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "wide_fingerprint_count": 104, + "wide_definition_bytes": 700000, + "lifecycle": ["initial import", "empty mirror resync", "warehouse retention reconciliation"] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index b236af6d7e..09c1495a90 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1296,12 +1296,12 @@ export interface components { /** Format: date-time */ observed_at: string; }; - /** @description A complete, ordered key-set scan of one physical state table and UTC observation day. */ + /** @description A complete, ordered key-set scan of one physical state table, optionally one UTC day. */ WarehouseReconciliationPage: { after?: null | components["schemas"]["WarehouseReconciliationCursor"]; complete: boolean; /** Format: date */ - day: string; + day?: string | null; /** Format: int32 */ page: number; /** Format: uuid */ diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 6036e52bf5..56395c504e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -7,6 +7,8 @@ use std::process::Stdio; use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; +use serde_json::json; +use sqlbuild_orchestration::api::constants::BODY_BYTES; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, DockerImageSource}; @@ -17,7 +19,9 @@ use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_pl use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink; use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; -use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, +}; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; @@ -26,11 +30,16 @@ use sqlbuild_orchestration::store::main::begin_offline_recovery::begin_offline_r use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryPage, WarehouseReconciliationCursor, WarehouseStateLocation, + WarehouseSyncResponse, +}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use tokio::process::{Child, Command}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinSet; -use tokio::time::MissedTickBehavior; +use tokio::time::{Instant, MissedTickBehavior}; use uuid::Uuid; use crate::argv::models::{Namespace, Stream, Value}; @@ -51,14 +60,16 @@ use crate::native_commands::_helpers::terminal::{ finish_lost_stdout, terminal_columns, write_output, }; use crate::native_commands::constants::{ - COLUMNS_VARIABLE, FALLBACK_COLUMNS, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, UNCAUGHT_EXCEPTION_STATUS, + UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, - LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, + LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, NativeCommand, + NativeInvocation, ProjectLocation, }; use crate::native_commands::types::LocationPublication; use crate::workers::main::docker_image_sources::docker_image_sources; @@ -199,7 +210,7 @@ pub(crate) async fn compile_generation( generation: i64, sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], stopped: impl Future>, -) -> Result>)>, ControlError> { +) -> Result, Vec)>, ControlError> { let (reply, publication) = oneshot::channel(); requests .send(LocationCompileRequest { @@ -231,8 +242,12 @@ pub(crate) async fn compile_generation( } }; let published = record.is_some(); - let refresh_at = record.as_ref().and_then(|(_, refresh_at)| *refresh_at); - if let Some((record, _)) = record { + let mut refresh_at = None; + let mut locations: Vec = Vec::new(); + if let Some((record, refresh, state_locations)) = record { + refresh_at = refresh + .map(|deadline| Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default()); + locations = state_locations; let output = serde_json::to_string(&record) .map_err(|error| ControlError::Invalid(error.to_string()))?; if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { @@ -242,7 +257,7 @@ pub(crate) async fn compile_generation( ))); } } - Ok(Some((published, refresh_at))) + Ok(Some((published, refresh_at, locations))) } /// The child is retained by its owning task while this exchange can be cancelled. @@ -266,6 +281,168 @@ pub(crate) async fn exchange_location_generation( }) } +/// The caller retains and reaps the child while a cancellable bounded exchange owns its pipes. +pub(crate) fn location_result( + (options, script): (&LocationServe, &Path), + (request, maximum_bytes): (Vec, u64), +) -> Result<(Child, impl Future>), ControlError> { + let mut child = location_runtime_child(options, script)?; + let mut stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location input is unavailable".into()))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("location output is unavailable".into()))?; + Ok((child, async move { + stdin.write_all(&request).await?; + stdin.write_all(b"\n").await?; + stdin.shutdown().await?; + drop(stdin); + let mut bytes: Vec = Vec::new(); + stdout + .take(maximum_bytes + 1) + .read_to_end(&mut bytes) + .await?; + if bytes.len() as u64 > maximum_bytes { + return Err(ControlError::Invalid( + "location reply exceeds its byte limit".into(), + )); + } + serde_json::from_slice(&bytes) + .map_err(|_| ControlError::Invalid("invalid location reply".into())) + })) +} + +pub(crate) fn metadata_sync( + (locations, generation): (Vec, i64), +) -> LocationMetadataSync { + LocationMetadataSync { + locations, + generation, + index: 0, + scan_id: Uuid::now_v7(), + page: 0, + source_after: None, + limit: MAX_HISTORY_RECORDS, + pending: None, + next_at: Instant::now(), + } +} + +pub(crate) fn history_request(state: &LocationMetadataSync) -> Result, ControlError> { + if state.index == 0 && state.page == 0 && state.limit == MAX_HISTORY_RECORDS { + drop(write_output(&[( + Stream::Stderr, + "Reading warehouse history for automatic synchronization...\n", + )])); + } + let kind = if state.index.is_multiple_of(2) { + "fingerprints" + } else { + "source_freshness" + }; + serde_json::to_vec(&json!({(kind): { + "location":state.locations[state.index / 2],"after":state.source_after,"observed_since":null,"limit":state.limit + }})).map_err(|error| ControlError::Invalid(error.to_string())) +} + +/// Reserve the largest mirror cursor before freezing: receipts prohibit later content splitting. +pub(crate) fn freeze_history_page( + mut state: LocationMetadataSync, + page: WarehouseHistoryPage, + poll: &LocationWorkPoll, +) -> Result { + let location = &state.locations[state.index / 2]; + let mut body = json!({"session_id":poll.session_id,"generation":state.generation,"records":page.records, + "reconciliation":{"scan_id":state.scan_id,"page":state.page,"state_database":location.database, + "state_schema":location.schema,"day":null,"complete":page.next_after.is_none(), + "after":WarehouseReconciliationCursor { observed_at: DateTime::::MAX_UTC, event_id: Uuid::nil() }}}); + if serde_json::to_vec(&body) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .len() + > BODY_BYTES + { + if state.limit > 1 { + state.limit = (state.limit / 2).max(1); + } else { + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history row exceeds the request bound; synchronization incomplete, no reconciliation authorized.\n", + )])); + } + } else { + body["reconciliation"]["after"] = serde_json::Value::Null; + state.pending = Some(LocationMetadataPage { + body, + next_source_after: page.next_after, + }); + } + Ok(state) +} + +pub(crate) async fn deliver_metadata_page( + (client, target): (&LocationWorkClient, &str), + body: &serde_json::Value, + fingerprints: bool, +) -> Result { + if fingerprints { + client + .sync_fingerprints( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid fingerprint batch"))?, + ) + .await + } else { + client + .sync_source_freshness( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid freshness batch"))?, + ) + .await + } +} + +/// Source progress waits for all mirror continuations of the identical frozen final batch. +pub(crate) fn acknowledge_metadata_page( + mut state: LocationMetadataSync, + response: WarehouseSyncResponse, +) -> Result { + let Some(mut pending) = state.pending.take() else { + return Ok(state); + }; + if let Some(after) = response.next_reconciliation_after { + pending.body["reconciliation"]["after"] = json!(after); + state.pending = Some(pending); + } else { + state.source_after = pending.next_source_after; + if state.source_after.is_some() { + state.page = state + .page + .checked_add(1) + .ok_or_else(|| ControlError::Invalid("warehouse scan page overflow".into()))?; + } else { + state.index += 1; + state.scan_id = Uuid::now_v7(); + state.page = 0; + state.limit = MAX_HISTORY_RECORDS; + if state.index == state.locations.len() * 2 { + state.index = 0; + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history synchronization completed.\n", + )])); + } + } + } + Ok(state) +} + /// Health IO remains native even while a user-code child holds its interpreter's GIL. pub(crate) async fn report_location_health( options: &LocationServe, @@ -386,6 +563,37 @@ pub(crate) fn location_runtime_child(options: &LocationServe, script: &Path) -> command.stdin(Stdio::piped()).stdout(Stdio::piped()).spawn() } +pub(crate) async fn check_published_location( + options: &LocationServe, + script: &Path, + (bearer, session_id, generation): (&str, Uuid, i64), + stopped: impl Future>, +) -> Result>, ControlError> { + drop(write_output(&[( + Stream::Stderr, + "Checking location connection with sqb debug...\n", + )])); + tokio::pin!(stopped); + let Some((connected, files)) = + check_location_connection(options, script, stopped.as_mut()).await? + else { + return Ok(None); + }; + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + if connected { + "Code location ready; connection check passed; watching for changes.\n" + } else { + "Code location published; connection check failed; watching for changes.\n" + }, + )])); + Ok(Some(files)) +} + /// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. pub(crate) async fn check_location_connection( options: &LocationServe, diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index e959a4b263..4006ab3e3e 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -93,5 +93,10 @@ pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; /// Reserve half the location work claim for acknowledgement retries after killing a callback. pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; +/// Independent read timeout and delay between complete automatic history scans. +pub(crate) const METADATA_POLL_SECONDS: u64 = 30; +/// A source-key cursor can repeat the last row's base64-encoded definition beside the records. +pub(crate) const METADATA_REPLY_BYTES: u64 = + 3 * sqlbuild_orchestration::api::constants::BODY_BYTES as u64; /// Bound piped terminal input independently of the credential owner's password policy. pub(crate) const PASSWORD_INPUT_BYTES: u64 = 4096; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs new file mode 100644 index 0000000000..3342da615b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -0,0 +1,122 @@ +//! Isolated read-only pages and exact acknowledgement retries share the published input fence. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::models::{LocationWorkClient, LocationWorkPoll}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryPage; +use tokio::time::Instant; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::{ + acknowledge_metadata_page, deliver_metadata_page, freeze_history_page, history_request, + location_result, +}; +use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::constants::{METADATA_POLL_SECONDS, METADATA_REPLY_BYTES}; +use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; + +/// One bounded source or mirror page leaves named work polling a turn between pages. +pub(super) async fn sync_location_metadata( + mut state: LocationMetadataSync, + (options, script, watch): (&LocationServe, &Path, &LocationWatch), + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + stopped: impl Future>, +) -> Result, ControlError> { + if Instant::now() < state.next_at + || state.locations.is_empty() + || location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed + { + return Ok(Some(state)); + } + tokio::pin!(stopped); + if state.pending.is_none() { + let Some(page) = read_metadata_page( + (options, script), + history_request(&state)?, + stopped.as_mut(), + ) + .await? + else { + return Ok(None); + }; + let Some(page) = page else { + state.limit = (state.limit / 2).max(1); + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", + )])); + return Ok(Some(state)); + }; + if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed + { + return Ok(Some(state)); + } + state = freeze_history_page(state, page, poll)?; + } + let Some(pending) = &state.pending else { + return Ok(Some(state)); + }; + let response = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + response = deliver_metadata_page((client, &options.target), &pending.body, state.index.is_multiple_of(2)) => response, + }; + let response = match response { + Ok(response) => response, + Err(error) => { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + &format!( + "Warehouse history delivery failed; retaining exact page for retry: {error}\n" + ), + )])); + return Ok(Some(state)); + } + }; + acknowledge_metadata_page(state, response).map(Some) +} + +/// An outer absence means shutdown; an inner absence is a failed read, never an empty table. +async fn read_metadata_page( + (options, script): (&LocationServe, &Path), + request: Vec, + stopped: impl Future>, +) -> Result>, ControlError> { + let (mut child, reply) = match location_result::>( + (options, script), + (request, METADATA_REPLY_BYTES), + ) { + Ok(exchange) => exchange, + Err(_) => return Ok(Some(None)), + }; + tokio::pin!(stopped); + let outcome = tokio::select! { + result = &mut stopped => { child.kill().await?; result?; return Ok(None); }, + result = tokio::time::timeout(Duration::from_secs(METADATA_POLL_SECONDS), async { + let page = reply.await?; + if !child.wait().await?.success() { + return Err(ControlError::Invalid("location metadata interpreter failed".into())); + } + Ok(page) + }) => result, + }; + if child.try_wait()?.is_none() { + child.kill().await?; + } + let page = match outcome { + Ok(Ok(page)) => page, + Ok(Err(_)) | Err(_) => None, + }; + Ok(Some(page)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs index 86c4585092..5c202c077d 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs @@ -6,10 +6,8 @@ use std::time::Duration; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; use sqlbuild_orchestration::store::errors::ControlError; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::process::Child; -use crate::native_commands::_helpers::execution::location_runtime_child; +use crate::native_commands::_helpers::execution::location_result; use crate::native_commands::constants::{SENSOR_EVALUATION_SECONDS, SENSOR_RESULT_BYTES}; use crate::native_commands::models::LocationServe; @@ -21,10 +19,11 @@ pub(super) async fn evaluate_location_sensor( ) -> Result, ControlError> { let request = serde_json::to_vec(evaluation).map_err(|error| ControlError::Invalid(error.to_string()))?; - let mut child = match location_runtime_child(options, script) { - Ok(child) => child, - Err(_) => return Ok(Some(CustomSensorOutcome::Failed)), - }; + let (mut child, reply) = + match location_result((options, script), (request, SENSOR_RESULT_BYTES)) { + Ok(exchange) => exchange, + Err(_) => return Ok(Some(CustomSensorOutcome::Failed)), + }; tokio::pin!(stopped); let outcome = tokio::select! { result = &mut stopped => { @@ -32,9 +31,15 @@ pub(super) async fn evaluate_location_sensor( result?; return Ok(None); }, - result = tokio::time::timeout(Duration::from_secs(SENSOR_EVALUATION_SECONDS), exchange_result(&mut child, request)) => { + result = tokio::time::timeout(Duration::from_secs(SENSOR_EVALUATION_SECONDS), async { + let outcome = reply.await?; + if !child.wait().await?.success() { + return Err(ControlError::Invalid("location sensor interpreter failed".into())); + } + Ok(outcome) + }) => { match result { - Ok(Ok((_, outcome))) => outcome, + Ok(Ok(outcome)) => outcome, Ok(Err(_)) => CustomSensorOutcome::Failed, Err(_) => CustomSensorOutcome::TimedOut, } @@ -45,32 +50,3 @@ pub(super) async fn evaluate_location_sensor( } Ok(Some(outcome)) } - -/// A bounded reply is accepted only after its interpreter exits successfully. -async fn exchange_result( - child: &mut Child, - request: Vec, -) -> Result<(&mut Child, CustomSensorOutcome), ControlError> { - let mut stdin = child - .stdin - .take() - .ok_or_else(|| ControlError::Invalid("sensor input is unavailable".into()))?; - let stdout = child - .stdout - .take() - .ok_or_else(|| ControlError::Invalid("sensor output is unavailable".into()))?; - stdin.write_all(&request).await?; - stdin.write_all(b"\n").await?; - stdin.shutdown().await?; - drop(stdin); - let mut bytes: Vec = Vec::new(); - let mut stdout = stdout.take(SENSOR_RESULT_BYTES + 1); - stdout.read_to_end(&mut bytes).await?; - if bytes.len() as u64 > SENSOR_RESULT_BYTES || !child.wait().await?.success() { - return Ok((child, CustomSensorOutcome::Failed)); - } - Ok(( - child, - serde_json::from_slice(&bytes).unwrap_or(CustomSensorOutcome::Failed), - )) -} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 5a8082353a..185efb131f 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -1,10 +1,9 @@ //! Coalesce source edits and retain the last good publication across isolated compiler failures. use std::future::Future; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::Duration; -use chrono::Utc; use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; use sqlbuild_orchestration::platform::models::{ LocationHealth, LocationWorkClient, LocationWorkPoll, @@ -16,16 +15,19 @@ use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - check_location_connection, compile_generation, location_generation, location_heartbeats, - report_location_health, shutdown_signal, + check_published_location, compile_generation, location_generation, location_heartbeats, + metadata_sync, report_location_health, shutdown_signal, }; use crate::native_commands::_helpers::paths::{ exclude_location_databases, location_watch, observe_location_inputs, }; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::main::_location_metadata::sync_location_metadata; use crate::native_commands::main::_location_work::run_location_work; -use crate::native_commands::models::{LocationCompileRequest, LocationServe}; +use crate::native_commands::models::{ + LocationCompileRequest, LocationMetadataSync, LocationServe, LocationWatch, +}; pub(super) async fn run_location( options: LocationServe, @@ -60,6 +62,7 @@ async fn watch_location( let mut watch = location_watch(&options.project_dir); let mut work_at = Instant::now() + Duration::from_secs(2); let mut published_generation = None; + let mut metadata = None; let mut clock = tokio::time::interval(Duration::from_millis(250)); clock.set_missed_tick_behavior(MissedTickBehavior::Skip); let stopped = shutdown_signal(); @@ -76,23 +79,17 @@ async fn watch_location( let ready; (watch, ready) = observe_location_inputs(&options.project_dir, watch); if !ready { - if Instant::now() >= work_at - && published_generation.is_some() - && watch.attempted.as_ref() == Some(&watch.observed) - { - let Some(files) = run_location_work( - (options, script), - (client, poll, &watch), - published_generation, - stopped.as_mut(), - ) - .await? - else { - return Ok(()); - }; - watch = exclude_location_databases(&options.project_dir, files, watch); - work_at = Instant::now() + Duration::from_secs(2); - } + let Some(idle) = poll_location( + (options, script), + (client, poll, published_generation), + (watch, work_at, metadata), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + (watch, work_at, metadata) = idle; continue; } published_generation = None; @@ -111,16 +108,15 @@ async fn watch_location( Stream::Stderr, &format!("Compiling location generation {generation} in an isolated interpreter...\n"), )])); - let Some((published, refresh_at)) = + let Some((published, refresh_at, locations)) = compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? else { return Ok(()); }; if published { - watch.retry_at = refresh_at.map(|deadline| { - Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default() - }); + watch.retry_at = refresh_at; published_generation = Some(generation); + metadata = Some(metadata_sync((locations, generation))); let Some(files) = check_published_location( options, script, @@ -146,33 +142,41 @@ async fn watch_location( } } -async fn check_published_location( - options: &LocationServe, - script: &Path, - (bearer, session_id, generation): (&str, Uuid, i64), +/// Named work and metadata share one idle turn without transferring interpreter ownership. +async fn poll_location( + (options, script): (&LocationServe, &Path), + (client, poll, generation): (&LocationWorkClient, &LocationWorkPoll, Option), + (mut watch, mut work_at, mut metadata): (LocationWatch, Instant, Option), stopped: impl Future>, -) -> Result>, ControlError> { - drop(write_output(&[( - Stream::Stderr, - "Checking location connection with sqb debug...\n", - )])); +) -> Result)>, ControlError> { tokio::pin!(stopped); - let Some((connected, files)) = - check_location_connection(options, script, stopped.as_mut()).await? - else { - return Ok(None); - }; - tokio::select! { - result = &mut stopped => { result?; return Ok(None); }, - () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, + if generation.is_some() && watch.attempted.as_ref() == Some(&watch.observed) { + if Instant::now() >= work_at { + let Some(files) = run_location_work( + (options, script), + (client, poll, &watch), + generation, + stopped.as_mut(), + ) + .await? + else { + return Ok(None); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + work_at = Instant::now() + Duration::from_secs(2); + } + if let Some(state) = metadata.take() { + metadata = sync_location_metadata( + state, + (options, script, &watch), + (client, poll), + stopped.as_mut(), + ) + .await?; + if metadata.is_none() { + return Ok(None); + } + } } - drop(write_output(&[( - Stream::Stderr, - if connected { - "Code location ready; connection check passed; watching for changes.\n" - } else { - "Code location published; connection check failed; watching for changes.\n" - }, - )])); - Ok(Some(files)) + Ok(Some((watch, work_at, metadata))) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 04602f8d37..fab61b5d4b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,5 +1,6 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; +pub(super) mod _location_metadata; pub(super) mod _location_sensor; pub(super) mod _location_work; pub(super) mod _run_admin; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 44eb36eec1..768f978eeb 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -8,6 +8,7 @@ use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use uuid::Uuid; use crate::argv::models::Namespace; @@ -173,6 +174,27 @@ pub(crate) struct LocationCompiler { pub(crate) python_sources: Vec, } +/// A delivered source page remains frozen until the API acknowledges it and any mirror cursor. +#[derive(Debug)] +pub(crate) struct LocationMetadataPage { + pub(crate) body: serde_json::Value, + pub(crate) next_source_after: Option>>, +} + +/// Only local retry/navigation state; durable scan receipts remain in the control store. +#[derive(Debug)] +pub(crate) struct LocationMetadataSync { + pub(crate) locations: Vec, + pub(crate) generation: i64, + pub(crate) index: usize, + pub(crate) scan_id: Uuid, + pub(crate) page: u32, + pub(crate) source_after: Option>>, + pub(crate) limit: usize, + pub(crate) pending: Option, + pub(crate) next_at: tokio::time::Instant, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-cli/src/native_commands/types.rs b/crates/sqlbuild-cli/src/native_commands/types.rs index 438f742268..862c4aa10d 100644 --- a/crates/sqlbuild-cli/src/native_commands/types.rs +++ b/crates/sqlbuild-cli/src/native_commands/types.rs @@ -2,6 +2,11 @@ use chrono::{DateTime, Utc}; use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; /// Publication metadata and the earliest calendar refresh deadline. -pub(crate) type LocationPublication = (SnapshotRecord, Option>); +pub(crate) type LocationPublication = ( + SnapshotRecord, + Option>, + Vec, +); diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index c5c94efdac..b360c21b28 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -1,6 +1,6 @@ //! Shared HTTP transport bounds for the server and native clients. -pub(crate) const BODY_BYTES: usize = 64 * 1024 * 1024; +pub const BODY_BYTES: usize = 64 * 1024 * 1024; /// Two 1024-byte passwords can each expand sixfold through JSON escaping. pub(crate) const PASSWORD_BODY_BYTES: usize = 16 * 1024; pub(crate) const PASSWORD_COOKIE: &str = "__Host-sqb-session"; diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs index 988baae351..46b8c6bfe2 100644 --- a/crates/sqlbuild-orchestration/src/api/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -1,7 +1,7 @@ //! Python-free HTTP endpoints; locations and workers have no direct database access. pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs index 12d55b10c5..8585964aaa 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs @@ -29,7 +29,7 @@ pub(crate) fn validate_batch(batch: &FingerprintBatch) -> Result<(), ControlErro batch.records.iter().any(|record| { record.state_database != page.state_database || record.state_schema != page.state_schema - || record.ts.date_naive() != page.day + || page.day.is_some_and(|day| record.ts.date_naive() != day) }) }); validate_fields( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs index bcaef6a86c..9967fd9183 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs @@ -28,7 +28,9 @@ pub(crate) fn validate_batch(batch: &SourceFreshnessBatch) -> Result<(), Control batch.records.iter().any(|record| { record.state_database != page.state_database || record.state_schema != page.state_schema - || record.observed_at.date_naive() != page.day + || page + .day + .is_some_and(|day| record.observed_at.date_naive() != day) }) }); validate_fields( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs index 986f9541b3..9c587565fb 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs @@ -20,12 +20,12 @@ pub(crate) async fn begin_reconciliation( mut batch: Value, ) -> Result { if page.state_schema.trim().is_empty() - || page.day.succ_opt().is_none() + || page.day.is_some_and(|day| day.succ_opt().is_none()) || (!page.complete && page.after.is_some()) - || page - .after - .as_ref() - .is_some_and(|after| after.observed_at.date_naive() != page.day) + || page.after.as_ref().is_some_and(|after| { + page.day + .is_some_and(|day| after.observed_at.date_naive() != day) + }) { return Err(ControlError::Invalid( "invalid reconciliation namespace, day or continuation".into(), @@ -118,40 +118,44 @@ pub(crate) async fn finish_reconciliation( if !page.complete { return Ok((transaction, response)); } - let end = page - .day - .succ_opt() - .ok_or_else(|| ControlError::Invalid("invalid reconciliation day".into()))?; - let start = page.day.and_time(NaiveTime::MIN).and_utc(); - let end = end.and_time(NaiveTime::MIN).and_utc(); let mut parameters = vec![ Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(page.state_schema.clone()), - Parameter::Time( - page.after - .as_ref() - .map_or(start, |cursor| cursor.observed_at), - ), - Parameter::Time(end), ]; - let database = match &page.state_database { + let mut bounds = vec![match &page.state_database { Some(database) => { parameters.push(Parameter::Text(database.clone())); - "state_database=$6" + format!("state_database=${}", parameters.len()) } - None => "state_database IS NULL", - }; - let lower_bound = if let Some(cursor) = &page.after { - parameters.push(Parameter::Id(cursor.event_id)); - format!("(observed_at,event_id)>($4,${})", parameters.len()) - } else { - "observed_at>=$4".into() - }; + None => "state_database IS NULL".into(), + }]; + if let Some(cursor) = &page.after { + let offset = parameters.len(); + parameters.extend([ + Parameter::Time(cursor.observed_at), + Parameter::Id(cursor.event_id), + ]); + bounds.push(format!( + "(observed_at,event_id)>(${},${})", + offset + 1, + offset + 2 + )); + } else if let Some(day) = page.day { + parameters.push(Parameter::Time(day.and_time(NaiveTime::MIN).and_utc())); + bounds.push(format!("observed_at>=${}", parameters.len())); + } + if let Some(day) = page.day { + let end = day + .succ_opt() + .ok_or_else(|| ControlError::Invalid("invalid reconciliation day".into()))?; + parameters.push(Parameter::Time(end.and_time(NaiveTime::MIN).and_utc())); + bounds.push(format!("observed_at<${}", parameters.len())); + } let candidates; (transaction, candidates) = query(transaction, &format!( "SELECT observed_at,event_id FROM {table} WHERE location_id=$1 AND target=$2 AND state_schema=$3 \ - AND {lower_bound} AND observed_at<$5 AND {database} ORDER BY observed_at,event_id LIMIT {MAX_HISTORY_RECORDS}"), parameters).await?; + AND {} ORDER BY observed_at,event_id LIMIT {MAX_HISTORY_RECORDS}", bounds.join(" AND ")), parameters).await?; if candidates.len() == MAX_HISTORY_RECORDS { response.next_reconciliation_after = Some( serde_json::from_value::(json!(candidates.last())) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index c112395089..ad85cc093e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -50,6 +50,7 @@ pub struct SourceFreshnessRead { pub after: Option<[Option; 11]>, /// Inclusive time bound, held fixed across pages; independent of the row-key cursor. pub observed_since: Option>, + pub limit: Option, } /// Fingerprints remain history here; latest-row selection belongs to the existing planner reader. @@ -84,7 +85,7 @@ pub struct FingerprintBatch { pub reconciliation: Option, } -/// A complete, ordered key-set scan of one physical state table and UTC observation day. +/// A complete, ordered key-set scan of one physical state table, optionally one UTC day. #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct WarehouseReconciliationPage { @@ -92,7 +93,7 @@ pub struct WarehouseReconciliationPage { pub page: u32, pub state_database: Option, pub state_schema: String, - pub day: NaiveDate, + pub day: Option, pub complete: bool, /// Repeat the identical final page with this mirror cursor until the response has none. pub after: Option, @@ -118,6 +119,7 @@ pub struct FingerprintRead { pub location: WarehouseStateLocation, pub after: Option<[Option; 13]>, pub observed_since: Option>, + pub limit: Option, } /// The private interpreter accepts only these two known state tables, never arbitrary SQL. diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs index 8fb2aac2cd..f0c9f222ff 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs @@ -236,6 +236,40 @@ pub(crate) async fn reconcile_history( before, "empty-copy resync preserves exact identities and observations" ); + let mut namespace = json!({"session_id":session_id,"generation":1,"records":[],"reconciliation":{ + "scan_id":Uuid::now_v7(),"page":0,"state_database":null,"state_schema":"reconcile_orders","day":null,"complete":false,"after":null}}); + post((client, &url, bearer), &namespace, StatusCode::OK).await; + assert_eq!( + observations(store, table).await, + before, + "partial namespace scan cannot delete" + ); + namespace["reconciliation"]["page"] = json!(1); + namespace["reconciliation"]["complete"] = json!(true); + for index in 0..3 { + let reply = post((client, &url, bearer), &namespace, StatusCode::OK).await; + namespace["reconciliation"]["after"] = reply["next_reconciliation_after"].clone(); + assert_eq!(namespace["reconciliation"]["after"].is_null(), index == 2); + } + let survivors = observations(store, table).await; + let mut ids: Vec<_> = survivors + .iter() + .map(|row| { + row["record_json"]["event_id"] + .as_str() + .expect("scoped control") + }) + .collect(); + ids.sort(); + assert_eq!( + ids, + [ + "reconciliation-other-database", + "reconciliation-other-schema", + "reconciliation-preview" + ], + "whole namespace includes both day boundaries, never other scopes" + ); retained_history( (store, table), (client, &url, bearer), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 8654e8d5f2..e46264c2ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -1,6 +1,6 @@ //! Compile in the location interpreter and publish data through its scoped HTTP credential. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::process::{Command, Stdio}; @@ -14,6 +14,7 @@ use sqlbuild_orchestration::declarations::models::{ Schedule, TriggerDeclaration, TriggerDefinition, }; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; @@ -173,7 +174,11 @@ pub(super) fn location_arguments<'py>( pub(super) fn publish_location( py: Python<'_>, arguments: &KeywordArguments<'_>, -) -> PyResult<(SnapshotRecord, Option>)> { +) -> PyResult<( + SnapshotRecord, + Option>, + Vec, +)> { if arguments .value("action")? .extract::>()? @@ -196,6 +201,18 @@ pub(super) fn publish_location( .extract::()?; let input = metadata(py, arguments, &directory)?; let (snapshot, _) = compile_location(py, arguments, &input)?; + let locations: BTreeSet<_> = snapshot + .input + .nodes + .values() + .filter_map(|node| { + let destination = &node.definition["destination"]; + Some(WarehouseStateLocation { + database: destination["database"].as_str().map(str::to_owned), + schema: destination["schema"].as_str()?.to_owned(), + }) + }) + .collect(); let triggers: BTreeMap = serde_json::from_value( snapshot .input @@ -229,7 +246,7 @@ pub(super) fn publish_location( PyValueError::new_err(message) })?; status(py, "Location snapshot published.")?; - Ok((record, refresh_at)) + Ok((record, refresh_at, locations.into_iter().collect())) } /// One-shot publication and the supervisor's private stdio runtime share the same compiler path. @@ -282,7 +299,7 @@ pub(crate) fn run_location<'py>( return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); } let publication = (|| { - let (record, _) = publish_location(py, &arguments)?; + let (record, _, _) = publish_location(py, &arguments)?; let output = serde_json::to_string(&record) .map_err(|error| PyValueError::new_err(error.to_string()))?; write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index e0da01f39b..3fcd248d91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -99,11 +99,12 @@ fn read_page( request: &WarehouseHistoryRead, ) -> PyResult { let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); - let (location, after, since, table, columns, timestamp) = match request { + let (location, after, since, limit, table, columns, timestamp) = match request { WarehouseHistoryRead::SourceFreshness(read) => ( &read.location, read.after.as_ref().map(|cursor| cursor.as_slice()), read.observed_since, + read.limit, SOURCE_FRESHNESS_TABLE_NAME, freshness_columns.as_slice(), OBSERVED_AT_COLUMN, @@ -112,11 +113,18 @@ fn read_page( &read.location, read.after.as_ref().map(|cursor| cursor.as_slice()), read.observed_since, + read.limit, FINGERPRINT_TABLE_NAME, FINGERPRINT_COLUMNS.as_slice(), COLUMN_TIMESTAMP, ), }; + let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); + if limit == 0 || limit > MAX_HISTORY_RECORDS { + return Err(PyValueError::new_err( + "metadata page limit is outside the supported bounds", + )); + } let database = location.database.as_ref().map_or_else( || py.None().into_bound(py), |value| PyString::new(py, value).into_any(), @@ -206,7 +214,7 @@ fn read_page( observed_since: observed_since.as_deref(), adapter: &adapter_name, has_event_id, - limit: MAX_HISTORY_RECORDS, + limit, }, ) .map_err(PyValueError::new_err)?; @@ -217,7 +225,7 @@ fn read_page( ("sql", PyString::new(py, &sql).into_any()), ( "limit", - pyo3::IntoPyObjectExt::into_bound_py_any(MAX_HISTORY_RECORDS, py)?, + pyo3::IntoPyObjectExt::into_bound_py_any(limit, py)?, ), ], )?; @@ -246,7 +254,7 @@ fn read_page( } next_after = Some(cursor); } - if read_count < MAX_HISTORY_RECORDS { + if read_count < limit { next_after = None; } Ok(WarehouseHistoryPage { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 34aa67f911..5305d54551 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -14,6 +14,9 @@ class SnapshotApi(NamedTuple): location_id: str bearer: str + def __repr__(self) -> str: + return f"SnapshotApi(url={self.url!r}, location_id={self.location_id!r}, bearer=)" + class ClaimedWorker(NamedTuple): run_id: str @@ -173,6 +176,17 @@ class CatalogConnectionTestCase: expected_run_id: str +@dataclass(frozen=True) +class AutomaticHistoryTestCase: + description: str + adapter_name: str + connection_key: str + expected_run_id: str + wide: bool = False + initial_seconds: int = 8 + expected_average_bytes: int = 0 + + @dataclass(frozen=True) class FingerprintHistoryTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index b318ee8bc7..a76820b8fa 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -1,8 +1,13 @@ """Real warehouse writers, populated predecessors and the isolated read-only history protocol.""" +import base64 +import gzip import hashlib import json +import os import sqlite3 +import sys +import time from contextlib import closing from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -31,6 +36,7 @@ TYPE_PROOF_FILES, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + AutomaticHistoryTestCase, CatalogConnectionTestCase, FingerprintHistoryTestCase, WarehouseHistoryTestCase, @@ -38,9 +44,11 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_contended_fingerprints, append_retried_fingerprints, + location_process, open_legacy_warehouse, snapshot_api, sync_warehouse_history, + wait_for_generation, warehouse_history_page, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -461,6 +469,143 @@ def test_given_external_warehouse_retention_when_a_complete_scan_arrives_then_mi assert not (project / ".sqlbuild/control.sqlite3").exists() +@pytest.mark.parametrize( + "test_case", + [ + AutomaticHistoryTestCase("DuckDB automatic history", "duckdb", "database", "legacy-run"), + AutomaticHistoryTestCase("SQLite automatic history", "sqlite", "path", "legacy-run"), + AutomaticHistoryTestCase( + "SQLite wide history pages", + "sqlite", + "path", + "legacy-run", + wide=True, + initial_seconds=30, + expected_average_bytes=640_000, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_warehouse_history_changes_then_periodic_read_only_sync_reconciles_without_manual_requests( + test_case: AutomaticHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" + ).read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + expected_runs: set[str] = {test_case.expected_run_id} + definition: str = base64.b64encode( + ("SELECT 17 /*" + "x" * fixture["wide_definition_bytes"] + "*/").encode() + ).decode() + for index in range(fixture["wide_fingerprint_count"] * test_case.wide): + expected_runs.add(f"automatic-{index}") + _ = adapter.execute( + connection=connection, + sql=( + "INSERT INTO _sqlbuild_fingerprints SELECT node_type," + f"'orders-{index}',target_database,target_schema,target_name,'automatic-{index}'," + "definition_hash,version_hash,schema_fingerprint," + f"'{definition}',metadata_json_b64,ts FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'" + ), + ) + assert adapter.query( + connection=connection, + sql="SELECT SUM(LENGTH(definition_b64))*3/4 FROM _sqlbuild_fingerprints", + limit=1, + ).rows[0][0] >= test_case.expected_average_bytes * len(expected_runs), ( + "wide fixture exceeds one API request before any synchronization" + ) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files( + project_dir=project, + files={"models/orders.sql": fixture["model"]}, + ) + with snapshot_api(tmp_path / "control") as api: + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + initial_tables: dict[str, set[str]] = { + "sqb_warehouse_fingerprints": expected_runs, + "sqb_warehouse_freshness": {test_case.expected_run_id}, + } + for expected_tables, seconds, mirror_sql, warehouse_sql in [ + ( + initial_tables, + test_case.initial_seconds, + "DELETE FROM {table}", + "SELECT 1", + ), + (initial_tables, 45, "SELECT 1", "DELETE FROM {table}"), + ({table: set() for table in initial_tables}, 45, "SELECT 1", "SELECT 1"), + ]: + deadline: float = time.monotonic() + seconds + observed: tuple[int, int] = (-1, -1) + expected: tuple[int, int] = ( + len(expected_tables["sqb_warehouse_fingerprints"]), + len(expected_tables["sqb_warehouse_freshness"]), + ) + while observed != expected: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with closing(sqlite3.connect(control)) as database: + observed = database.execute( + "SELECT (SELECT COUNT(*) FROM sqb_warehouse_fingerprints)," + "(SELECT COUNT(*) FROM sqb_warehouse_freshness)" + ).fetchone() + time.sleep(0.05) + with closing(sqlite3.connect(control)) as database: + bodies: list[tuple[bytes]] = database.execute( + "SELECT t.body FROM sqb_warehouse_fingerprints f " + "JOIN sqb_source_texts t ON t.text_hash=f.definition_text_hash" + ).fetchall() + assert sum(len(gzip.decompress(body)) for (body,) in bodies) >= ( + test_case.expected_average_bytes + * len(expected_tables["sqb_warehouse_fingerprints"]) + ) + for table, expected_runs in expected_tables.items(): + rows: list[tuple[str]] = database.execute( + f"SELECT record_json FROM {table}" + ).fetchall() + runs: set[str] = {json.loads(row[0])["run_id"] for row in rows} + assert runs == expected_runs + _ = database.execute(mirror_sql.format(table=table)) + database.commit() + connection = adapter.connect( + {test_case.connection_key: str(project / "warehouse.db")} + ) + for table in ("_sqlbuild_fingerprints", "_sqlbuild_source_freshness"): + _ = adapter.execute( + connection=connection, sql=warehouse_sql.format(table=table) + ) + adapter.close(connection) + + @pytest.mark.parametrize( "test_case", [ From 96821ffbd980f9c474537dbed4e2bcccf2f62b6f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 07:33:16 +0000 Subject: [PATCH 086/138] feat: add permission-protected ordered security audit feed --- .../orchestration/security_audit_order.json | 7 + clients/typescript/src/schema.d.ts | 80 ++++ .../authentication/password_sessions.rs | 4 + .../src/api/_helpers/resources/audit.rs | 32 ++ .../src/api/_helpers/resources/mod.rs | 1 + .../sqlbuild-orchestration/src/api/models.rs | 12 +- .../src/authorization/main/mod.rs | 1 + .../authorization/main/read_security_audit.rs | 40 ++ .../src/credentials/main/append_audit.rs | 17 +- .../src/credentials/models.rs | 16 + .../src/store/_helpers/migration.rs | 1 + .../src/store/audit_storage_order.sql | 9 + .../src/store/main/migrate_store.rs | 11 +- .../tests/credentials/test_audit.rs | 371 ++++++++++++++++++ .../tests/credentials/test_contracts.rs | 2 + .../tests/credentials/test_types.rs | 6 + .../tests/store/helpers.rs | 3 + .../publication/person/helpers.py | 23 ++ .../publication/test_person_journey.py | 8 +- 19 files changed, 631 insertions(+), 13 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs create mode 100644 crates/sqlbuild-orchestration/src/store/audit_storage_order.sql create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_audit.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json b/benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json new file mode 100644 index 0000000000..6e80dd8d79 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json @@ -0,0 +1,7 @@ +{ + "occurred_at": [ + "2026-02-02T12:00:00Z", + "2020-01-01T00:00:00Z", + "2026-02-01T12:00:00Z" + ] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 09c1495a90..ef5da4af9e 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -147,6 +147,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/audit": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readSecurityAudit"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -1073,6 +1089,24 @@ export interface components { refresh_at: string; times: string[]; }; + /** @description Durable security facts with an exclusive storage-order cursor. */ + SecurityAuditEvent: { + action: string; + actor: string; + after_json: unknown; + before_json: unknown; + /** Format: uuid */ + event_id: string; + granting_role: string; + ip_address?: string | null; + /** Format: date-time */ + occurred_at: string; + /** Format: uuid */ + request_id: string; + scope_json: unknown; + /** Format: int64 */ + storage_id: number; + }; SensorCondition: { /** @enum {string} */ kind: "platform_event"; @@ -1627,6 +1661,52 @@ export interface operations { }; }; }; + readSecurityAudit: { + parameters: { + query?: { + /** @description Exclusive storage-order cursor; pre-upgrade facts are ordered by timestamp and event ID */ + after?: number; + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SecurityAuditEvent"][]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 16f92c1751..212171f999 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -30,6 +30,10 @@ pub(crate) fn protected(store: &ControlStore) -> Router { Router::new() .route("/api/auth/session", get(current)) .route("/api/auth/logout", post(logout)) + .route( + "/api/control/audit", + get(crate::api::_helpers::resources::audit::read), + ) .route( "/api/admin/people/{principal_id}", post(crate::api::_helpers::resources::administrators::change), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs new file mode 100644 index 0000000000..4972c6f79c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs @@ -0,0 +1,32 @@ +//! Person sessions read bounded security facts without machine-token or ambient-viewer access. + +use axum::extract::{Query, State}; +use axum::http::HeaderMap; +use axum::{Extension, Json}; + +use crate::api::_helpers::authentication::password_transport::session_credentials; +use crate::api::errors::ApiError; +use crate::api::models::ReadAuditQuery; +use crate::authorization::main::read_security_audit::read_security_audit; +use crate::credentials::models::{AuditContext, SecurityAuditEvent}; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/control/audit", operation_id = "readSecurityAudit", + params(("after" = Option, Query, description = "Exclusive storage-order cursor; pre-upgrade facts are ordered by timestamp and event ID"), ("limit" = Option, Query)), + responses((status = 200, body = Vec), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = [])))] +pub(crate) async fn read( + (State(store), Extension(context)): (State, Extension), + Query(options): Query, + headers: HeaderMap, +) -> Result>, ApiError> { + Ok(Json( + read_security_audit( + &store, + (&context, session_credentials(&headers)), + (options.after.unwrap_or(0), options.limit.unwrap_or(100)), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 80a79e5e21..cbb0575196 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod administrators; +pub(crate) mod audit; pub(crate) mod changes; pub(crate) mod client; pub(crate) mod docker_image; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 69ebcebf67..7c08ea2c40 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -6,14 +6,15 @@ use crate::api::_helpers::authentication::oidc; use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::trusted_proxy; use crate::api::_helpers::resources::{ - administrators, changes, location_health, location_work, person_operations, run_reads, runs, - snapshots, trigger_reads, warehouse_metadata, + administrators, audit, changes, location_health, location_work, person_operations, run_reads, + runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] #[openapi(paths( administrators::change, administrators::assign, + audit::read, person_operations::request_job, person_operations::cancel_run, person_operations::rerun, @@ -58,6 +59,13 @@ pub(crate) struct ReadQuery { pub(crate) limit: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReadAuditQuery { + pub(crate) after: Option, + pub(crate) limit: Option, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RunRecallQuery { diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs index 5c65c595fe..36969020a7 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod authorize_session; pub mod bootstrap_user; pub mod change_administrator; pub(crate) mod deny_authorization; +pub mod read_security_audit; pub mod recover_administrator; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs b/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs new file mode 100644 index 0000000000..d03d190806 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs @@ -0,0 +1,40 @@ +//! Live installation-wide audit permission and each bounded page share one transaction. + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, PermissionScope, SecurityAuditEvent, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn read_security_audit( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + (after, limit): (i64, u32), +) -> Result, ControlError> { + let (transaction, _) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::AuditView, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + if after < 0 || !(1..=100).contains(&limit) { + return Err(ControlError::Invalid( + "audit pages require a nonnegative storage cursor and a limit between 1 and 100".into(), + )); + } + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,actor,action,granting_role,request_id,ip_address,scope_json,before_json,after_json \ + FROM sqb_security_audit WHERE storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(i64::from(limit))]).await?; + let events = rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + Ok(events) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs index 30b2c7362d..41bcc482b7 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs @@ -19,13 +19,24 @@ pub(crate) async fn append_audit( "audit actor and granting role are required".into(), )); } + let (transaction, rows) = query( + transaction, + "SELECT COALESCE(MAX(storage_id),0) AS last_storage_id FROM sqb_security_audit", + vec![], + ) + .await?; + let storage_id = rows + .first() + .and_then(|row| row["last_storage_id"].as_i64()) + .and_then(|last| last.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("audit storage order is exhausted".into()))?; let now = transaction.now; let (transaction, _) = query(transaction, - "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, request_id, ip_address, scope_json, before_json, after_json) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, request_id, ip_address, scope_json, before_json, after_json, storage_id) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", vec![Parameter::Id(event_id), Parameter::Time(now), Parameter::Text(context.actor.clone()), Parameter::Text(action.into()), Parameter::Text(context.granting_role.clone()), Parameter::Id(context.request_id), Parameter::OptionalText(context.ip_address.map(|value| value.to_string())), - Parameter::Json(scope), Parameter::Json(before), Parameter::Json(after)]).await?; + Parameter::Json(scope), Parameter::Json(before), Parameter::Json(after), Parameter::Integer(storage_id)]).await?; Ok(transaction) } diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index 05025f4def..4a3fcc411f 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -38,6 +38,22 @@ pub struct AuditContext { pub ip_address: Option, } +/// Durable security facts with an exclusive storage-order cursor. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SecurityAuditEvent { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub actor: String, + pub action: String, + pub granting_role: String, + pub request_id: Uuid, + pub ip_address: Option, + pub scope_json: serde_json::Value, + pub before_json: serde_json::Value, + pub after_json: serde_json::Value, +} + /// Closed permission vocabulary; unknown actions fail deserialization. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub enum Action { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 62ffc10aad..2068c2d4ff 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -26,6 +26,7 @@ pub(crate) const ADMINISTRATOR_INITIALIZATION: &str = include_str!("../administrator_initialization.sql"); pub(crate) const EXTERNAL_SESSIONS: &str = include_str!("../external_sessions.sql"); pub(crate) const WAREHOUSE_RECONCILIATION: &str = include_str!("../warehouse_reconciliation.sql"); +pub(crate) const AUDIT_STORAGE_ORDER: &str = include_str!("../audit_storage_order.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/audit_storage_order.sql b/crates/sqlbuild-orchestration/src/store/audit_storage_order.sql new file mode 100644 index 0000000000..68a2ff99fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/audit_storage_order.sql @@ -0,0 +1,9 @@ +ALTER TABLE sqb_security_audit ADD COLUMN storage_id BIGINT; +WITH ordered AS ( + SELECT event_id, ROW_NUMBER() OVER (ORDER BY occurred_at,event_id) AS storage_id + FROM sqb_security_audit +) +UPDATE sqb_security_audit SET storage_id=( + SELECT ordered.storage_id FROM ordered WHERE ordered.event_id=sqb_security_audit.event_id +); +CREATE UNIQUE INDEX sqb_security_audit_storage_id ON sqb_security_audit(storage_id); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 239267fbb4..b13dfa4120 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,11 +4,11 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - ADMINISTRATOR_INITIALIZATION, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, - EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, - OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, - SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_RECONCILIATION, render, - verify_migrations, + ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, + DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, + NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, + WAREHOUSE_RECONCILIATION, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -50,6 +50,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("administrator_initialization", ADMINISTRATOR_INITIALIZATION), ("external_sessions", EXTERNAL_SESSIONS), ("warehouse_reconciliation", WAREHOUSE_RECONCILIATION), + ("audit_storage_order", AUDIT_STORAGE_ORDER), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs new file mode 100644 index 0000000000..77a02110b4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -0,0 +1,371 @@ +//! Real durable audit pages retain clock-independent order, live permissions and atomic writes. + +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, + read_security_audit::read_security_audit, +}; +use crate::credentials::main::{ + append_audit::append_audit, create_principal::create_principal, login_password::login_password, + set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorAuthentication, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, + PrincipalKind, RoleAssignmentRequest, +}; +use crate::credentials::tests::test_types::AuditTestCase; +use crate::store::errors::ControlError; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_and_atomicity_hold() +{ + let test_cases = [ + AuditTestCase { + description: "SQLite durable audit pages", + backend: Backend::Sqlite, + expected_page_length: 2, + }, + AuditTestCase { + description: "PostgreSQL durable audit pages", + backend: Backend::Postgres, + expected_page_length: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("local peer")), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = + create_principal(store, &context, ("orders-person", PrincipalKind::Person)) + .await + .expect("person"); + let mut sessions = Vec::new(); + for (principal, name) in [(admin, "orders-admin"), (person, "orders-person")] { + set_password( + store, + &context, + PasswordAuthority::Operator(principal), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid login"); + }; + sessions.push(session); + } + let admin_credentials = (sessions[0].bearer.as_str(), ""); + let ordinary_credentials = (sessions[1].bearer.as_str(), ""); + for role in [ + BuiltinRole::Viewer, + BuiltinRole::Operator, + BuiltinRole::Developer, + ] { + assert!( + assign_builtin_role( + store, + ( + &context, + (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()) + ), + &RoleAssignmentRequest { + principal_id: person, + role, + location_id: None, + target: None, + revoke: false, + }, + &AdministratorAuthentication::Password + ) + .await + .expect("role assignment") + ); + assert!(matches!( + read_security_audit(&replica, (&context, ordinary_credentials), (0, 100)).await, + Err(ControlError::Denied { .. }) + )); + } + let existing = read_security_audit(store, (&context, admin_credentials), (0, 100)) + .await + .expect("existing security facts"); + let mut expected = existing + .iter() + .map(|event| (event.occurred_at, event.event_id)) + .collect::>(); + expected.sort(); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("predecessor"); + for sql in [ + "DROP INDEX sqb_security_audit_storage_id", + "ALTER TABLE sqb_security_audit DROP COLUMN storage_id", + "DELETE FROM sqb_schema_migrations WHERE name='audit_storage_order'", + ] { + (transaction, _) = query(transaction, sql, vec![]) + .await + .expect("populated predecessor"); + } + commit(transaction).await.expect("commit predecessor"); + migrate_store(&replica) + .await + .expect("additive audit upgrade"); + migrate_store(store) + .await + .expect("idempotent audit upgrade"); + let upgraded = read_security_audit(&replica, (&context, admin_credentials), (0, 100)) + .await + .expect("backfilled audit"); + assert_eq!( + upgraded + .iter() + .map(|event| event.event_id) + .collect::>(), + expected.into_iter().map(|(_, id)| id).collect::>() + ); + assert_eq!( + upgraded + .iter() + .map(|event| event.storage_id) + .collect::>(), + (1..=upgraded.len() as i64).collect::>() + ); + let cursor = upgraded.last().expect("populated history").storage_id; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json", + ); + let inputs: Value = serde_json::from_str( + &std::fs::read_to_string(path).expect("DuckShop clock fixture"), + ) + .expect("clock inputs"); + let mut ids = Vec::new(); + for (index, timestamp) in inputs["occurred_at"] + .as_array() + .expect("clock steps") + .iter() + .enumerate() + { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("append clock fact"); + transaction.now = timestamp + .as_str() + .expect("timestamp") + .parse::>() + .expect("clock value"); + let id = Uuid::now_v7(); + ids.push(id); + let transaction = append_audit( + transaction, + &context, + (id, "audit.cursor_fixture"), + ( + json!({"observation":index}), + json!(null), + json!({"recorded":true}), + ), + ) + .await + .expect("durable clock fact"); + commit(transaction).await.expect("commit clock fact"); + } + let first = read_security_audit(&replica, (&context, admin_credentials), (cursor, 2)) + .await + .expect("first bounded page"); + assert_eq!( + first.len(), + test_case.expected_page_length, + "{}", + test_case.description + ); + assert_eq!( + first.iter().map(|event| event.event_id).collect::>(), + ids[..2] + ); + assert!(first[0].occurred_at > first[1].occurred_at); + assert_eq!(first[0].storage_id, cursor + 1); + assert_eq!(first[1].storage_id, cursor + 2); + assert_eq!(first[1].actor, context.actor); + assert_eq!(first[1].granting_role, context.granting_role); + assert_eq!(first[1].request_id, context.request_id); + assert_eq!(first[1].ip_address.as_deref(), Some("127.0.0.1")); + assert_eq!(first[1].scope_json, json!({"observation":1})); + assert_eq!(first[1].before_json, json!(null)); + assert_eq!(first[1].after_json, json!({"recorded":true})); + let last = read_security_audit( + store, + (&context, admin_credentials), + (first[1].storage_id, 2), + ) + .await + .expect("strict continuation"); + assert_eq!( + last.iter().map(|event| event.event_id).collect::>(), + ids[2..] + ); + let cursor = last[0].storage_id; + assert!( + read_security_audit(store, (&context, admin_credentials), (cursor, 100)) + .await + .expect("empty continuation") + .is_empty() + ); + for bounds in [(-1, 1), (0, 0), (0, 101)] { + assert!(matches!( + read_security_audit(store, (&context, admin_credentials), bounds).await, + Err(ControlError::Invalid(_)) + )); + } + let fault = TransactionFault { + description: "audit append fails atomically", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'audit unavailable')", + postgres_body: "RAISE EXCEPTION 'audit unavailable'", + }; + install_fault(store, &fault).await; + assert!( + create_principal(store, &context, ("orders-rollback", PrincipalKind::Person)) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + read_security_audit(&replica, (&context, admin_credentials), (cursor, 100)) + .await + .expect("failed audit did not publish") + .is_empty() + ); + create_principal(store, &context, ("orders-rollback", PrincipalKind::Person)) + .await + .expect("failed principal did not persist"); + let after = read_security_audit(store, (&context, admin_credentials), (cursor, 100)) + .await + .expect("successful retry"); + assert_eq!(after.len(), 1); + assert_eq!(after[0].storage_id, cursor + 1); + let cursor = after[0].storage_id; + let (left, right) = tokio::join!( + create_principal( + store, + &context, + ("orders-concurrent", PrincipalKind::Person) + ), + create_principal( + &replica, + &context, + ("customers-concurrent", PrincipalKind::Person) + ), + ); + assert_ne!( + left.expect("first concurrent append"), + right.expect("second concurrent append") + ); + let concurrent = + read_security_audit(store, (&context, admin_credentials), (cursor, 100)) + .await + .expect("serialized writers"); + assert_eq!( + concurrent + .iter() + .map(|event| event.storage_id) + .collect::>(), + vec![cursor + 1, cursor + 2] + ); + assert!( + concurrent + .iter() + .all(|event| event.action == "principal.created") + ); + assert!( + assign_builtin_role( + store, + ( + &context, + (sessions[0].bearer.as_str(), sessions[0].csrf.as_str()) + ), + &RoleAssignmentRequest { + principal_id: person, + role: BuiltinRole::Admin, + location_id: None, + target: None, + revoke: false, + }, + &AdministratorAuthentication::Password + ) + .await + .expect("second administrator") + ); + assert!( + assign_builtin_role( + store, + ( + &context, + (sessions[1].bearer.as_str(), sessions[1].csrf.as_str()) + ), + &RoleAssignmentRequest { + principal_id: admin, + role: BuiltinRole::Admin, + location_id: None, + target: None, + revoke: true, + }, + &AdministratorAuthentication::Password + ) + .await + .expect("live role revocation") + ); + assert!(matches!( + read_security_audit(&replica, (&context, admin_credentials), (0, 100)).await, + Err(ControlError::Denied { .. }) + )); + assert!( + !read_security_audit(&replica, (&context, ordinary_credentials), (0, 100)) + .await + .expect("replacement administrator reads audit") + .is_empty() + ); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + "orders-replacement-2026", + ) + .await + .expect("revoke old sessions"); + assert!(matches!( + read_security_audit(&replica, (&context, ordinary_credentials), (0, 100)).await, + Err(ControlError::Denied { .. }) + )); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index 23bd61658c..0cfe559ea6 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -6,6 +6,8 @@ pub(crate) mod helpers; pub(crate) mod location_helpers; #[path = "test_administrators.rs"] mod test_administrators; +#[path = "test_audit.rs"] +mod test_audit; #[path = "test_contract.rs"] mod test_contract; #[path = "passwords/test_contract.rs"] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index c9e7f3cb76..e7d8036e37 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -18,3 +18,9 @@ pub(crate) struct AdministratorRecoveryTestCase { pub(crate) backend: Backend, pub(crate) expected_recoveries: usize, } + +pub(crate) struct AuditTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_page_length: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 3ea401f5e1..68d011be0e 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,9 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP INDEX sqb_security_audit_storage_id", + "ALTER TABLE sqb_security_audit DROP COLUMN storage_id", + "DELETE FROM sqb_schema_migrations WHERE name='audit_storage_order'", "DROP TABLE sqb_warehouse_scan_keys", "DROP TABLE sqb_warehouse_scan_pages", "DROP INDEX sqb_fingerprints_reconciliation", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 34fd0c62ac..e2ce99fe97 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -7,8 +7,11 @@ from http.cookies import SimpleCookie from pathlib import Path from typing import Any +from urllib.error import HTTPError from urllib.request import Request, urlopen +import pytest + from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( SnapshotApi, ) @@ -56,6 +59,26 @@ def person_read(api: SnapshotApi, path: str, *, session: tuple[str, str]) -> Any return json.load(response) +def verify_security_audit( + api: SnapshotApi, administrator: tuple[str, str], ordinary: tuple[str, str] +) -> None: + audit: list[dict[str, Any]] = person_read( + api, "/api/control/audit?limit=3", session=administrator + ) + assert len(audit) == 3 + cursor: int = audit[-1]["storage_id"] + following: list[dict[str, Any]] = person_read( + api, f"/api/control/audit?after={cursor}&limit=3", session=administrator + ) + assert len(following) == 3 + assert all(event["storage_id"] > cursor for event in following) + assert len({event["event_id"] for event in audit + following}) == 6 + assert all(event["actor"] and event["granting_role"] for event in audit + following) + with pytest.raises(HTTPError) as audit_rejected: + person_read(api, "/api/control/audit", session=ordinary) + assert audit_rejected.value.code == 403 + + def person_request( api: SnapshotApi, path: str, *, body: Any, session: tuple[str, str] = ("", "") ) -> Any: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index ee4242c6e0..ab522a37e7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -30,6 +30,7 @@ person_read, person_request, provision_person, + verify_security_audit, ) @@ -82,6 +83,7 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi ) admin_session: tuple[str, str] = person_login(worker.api, admin["name"]) session: tuple[str, str] = person_login(worker.api, person["name"]) + verify_security_audit(worker.api, admin_session, session) runs_path: str = f"/api/control/locations/{worker.api.location_id}/targets/development/runs" request: dict[str, Any] = { "run_id": str(uuid4()), @@ -162,10 +164,10 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { "applied": True } - manual: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, run_id=request["run_id"] + assert ( + wait_for_run(worker.api, worker.process, worker.log, run_id=request["run_id"])["status"] + == test_case.expected_status ) - assert manual["status"] == test_case.expected_status assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { "applied": False } From 0da04d4c6205aadbf247e609bcde0649ff9df480 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 07:33:28 +0000 Subject: [PATCH 087/138] fix: wait briefly for DuckDB cross-process file locks --- .../local_catalog_contention.json | 6 ++ .../contract/classes/duckdb_backed_adapter.py | 26 +++++- .../sqlbuild/adapters/duckdb/_test_types.py | 10 +++ .../sqlbuild/adapters/duckdb/test_client.py | 80 +++++++++++++++++++ 4 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json b/benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json new file mode 100644 index 0000000000..d14975603b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json @@ -0,0 +1,6 @@ +{ + "setup_sql": "CREATE TABLE orders AS SELECT 23 AS amount", + "query": "SELECT amount FROM orders", + "release_after_seconds": 0.5, + "persistent_release_after_seconds": 6.0 +} diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index f6d52b95c4..fb8d0e8fe5 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -6,6 +6,7 @@ import json import logging import re +import time from datetime import date, datetime from decimal import Decimal from importlib.machinery import ModuleSpec @@ -1075,14 +1076,31 @@ def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> return f"'{value}'" def connect(self, config: dict[str, Any]) -> Any: - """Open a DuckDB connection from the resolved connection config.""" + """Open DuckDB, waiting up to five seconds for another process's database file lock.""" import duckdb database: str = str(config.get("database", ":memory:")) - raw_connection: duckdb.DuckDBPyConnection = duckdb.connect( - database=database, read_only=bool(config.get("read_only", False)) - ) + deadline: float = time.monotonic() + 5 + while True: + try: + raw_connection: duckdb.DuckDBPyConnection = duckdb.connect( + database=database, read_only=bool(config.get("read_only", False)) + ) + break + except duckdb.IOException as error: + if ( + not any( + marker in str(error) + for marker in ( + ": Conflicting lock is held in ", + "\nFile is already open in ", + ) + ) + or time.monotonic() >= deadline + ): + raise + time.sleep(0.05) return self._configured_connection(raw_connection=raw_connection, config=config) def connect_local_replay(self, database: Path) -> Any: diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index 492b5c02ef..d6b97a4713 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -1,3 +1,4 @@ +from contextlib import AbstractContextManager from dataclasses import dataclass, field from sqlbuild._native import InferredNullability, SeedCsvSettings @@ -41,6 +42,15 @@ class ConnectTestCase: expected_connects: bool +@dataclass(frozen=True) +class ContendedConnectTestCase: + description: str + read_only: bool + release_key: str + expected_connection: AbstractContextManager[object] + expected_rows: list[tuple[int]] + + @dataclass(frozen=True) class ConnectSettingsTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index d7c27e5bf5..607354a7fb 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -1,7 +1,12 @@ from __future__ import annotations +import json +import subprocess +import sys +from contextlib import nullcontext from decimal import Decimal from pathlib import Path +from threading import Timer from typing import Any import duckdb @@ -32,6 +37,7 @@ CompositeRowDiffSamplingTestCase, ConnectSettingsTestCase, ConnectTestCase, + ContendedConnectTestCase, DeleteInsertTestCase, DiffRowsErrorTestCase, DiffRowsTestCase, @@ -172,6 +178,80 @@ def test_given_config_when_connecting_then_returns_usable_connection( adapter.close(connection) +@pytest.mark.parametrize( + "test_case", + [ + ContendedConnectTestCase( + description="writer waits for a catalog reader", + read_only=False, + release_key="release_after_seconds", + expected_connection=nullcontext(), + expected_rows=[(23,)], + ), + ContendedConnectTestCase( + description="catalog reader waits for a writer", + read_only=True, + release_key="release_after_seconds", + expected_connection=nullcontext(), + expected_rows=[(23,)], + ), + ContendedConnectTestCase( + description="persistent contention remains an error", + read_only=False, + release_key="persistent_release_after_seconds", + expected_connection=pytest.raises(duckdb.IOException, match="lock|already open"), + expected_rows=[(23,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_catalog_contention_when_connecting_then_temporary_locks_wait_and_persistent_locks_fail( + test_case: ContendedConnectTestCase, adapter: DuckDbAdapter, tmp_path: Path +) -> None: + inputs: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json" + ).read_text() + ) + database: Path = tmp_path / "orders.duckdb" + connection: Any = adapter.connect({"database": str(database)}) + adapter.execute(connection=connection, sql=inputs["setup_sql"]) + adapter.close(connection) + with subprocess.Popen( + [ + sys.executable, + "-c", + "import duckdb,sys; c=duckdb.connect(sys.argv[1],read_only=sys.argv[2]=='true'); " + "print('ready',flush=True); sys.stdin.read(1); c.close()", + str(database), + str(not test_case.read_only).lower(), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) as holder: + assert holder.stdout is not None + assert holder.stdout.readline().strip() == "ready" + release: Timer = Timer( + inputs[test_case.release_key], + holder.communicate, + kwargs={"input": "\n", "timeout": 5}, + ) + release.start() + try: + with test_case.expected_connection: + connection = adapter.connect( + {"database": str(database), "read_only": test_case.read_only} + ) + try: + assert connection.execute(inputs["query"]).fetchall() == test_case.expected_rows + finally: + adapter.close(connection) + finally: + release.join() + assert holder.returncode == 0 + + @pytest.mark.parametrize( "test_case", [ From ddbe0026e44c44bbc46f652e1354784c4db1b244 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:09:24 +0000 Subject: [PATCH 088/138] feat: incrementally synchronize warehouse history --- .../incremental_warehouse_history.json | 17 +++ clients/typescript/src/schema.d.ts | 90 ++++++++++++ .../src/native_commands/_helpers/execution.rs | 37 ++++- .../src/native_commands/constants.rs | 5 +- .../main/_location_metadata.rs | 72 +++++++--- .../src/native_commands/main/_run_location.rs | 5 +- .../src/native_commands/models.rs | 6 + .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 22 ++- .../sqlbuild-orchestration/src/api/models.rs | 3 +- .../src/platform/_helpers/work_client.rs | 14 +- .../src/warehouse_metadata/main/mod.rs | 1 + .../warehouse_metadata/main/read_watermark.rs | 42 ++++++ .../src/warehouse_metadata/models.rs | 22 ++- .../tests/api/warehouse_metadata/helpers.rs | 135 +++++++++++++++++- .../compiled_project/publication/helpers.py | 9 ++ .../publication/test_warehouse_metadata.py | 98 ++++++++++++- 17 files changed, 551 insertions(+), 31 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json new file mode 100644 index 0000000000..8bb1b8bd65 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -0,0 +1,17 @@ +{ + "initial": [{"run_id": "watermark-run", "observed_at": "2026-02-01 12:00:00"}], + "stages": [ + {"arrivals": [], "overlap_count": 0, "completed_cycles": 1, "mirror_sql": "SELECT 1", "expected_runs": ["legacy-run", "watermark-run"]}, + {"arrivals": [ + {"run_id": "a-future", "observed_at": "2026-02-01 12:01:00"}, + {"run_id": "within-overlap", "observed_at": "2026-02-01 11:58:00"}, + {"run_id": "zz-boundary", "observed_at": "2026-02-01 11:55:00"}, + {"run_id": "before-overlap", "observed_at": "2026-02-01 11:54:00"} + ], "overlap_count": 104, "completed_cycles": 2, "mirror_sql": "SELECT 1", "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future"]}, + {"arrivals": [], "overlap_count": 0, "completed_cycles": 3, "mirror_sql": "DELETE FROM {table}", "expected_overlap_count": 104, "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future", "before-overlap"]} + ], + "insert_sql": [ + "INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'", + "INSERT INTO _sqlbuild_source_freshness SELECT source_name,target_database,target_schema,target_name,'{run_id}',strategy,value_kind,data_version,data_version_hash,'{observed_at}' FROM _sqlbuild_source_freshness WHERE run_id='legacy-run'" + ] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ef5da4af9e..bd486ace2e 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -307,6 +307,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/watermark": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["watermark"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -1324,6 +1340,11 @@ export interface components { target_schema?: string | null; value_kind: string; }; + /** + * @description Only immutable history tables support a timestamp watermark. + * @enum {string} + */ + WarehouseHistoryKind: "fingerprints" | "source_freshness"; WarehouseReconciliationCursor: { /** Format: uuid */ event_id: string; @@ -1343,10 +1364,24 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ + WarehouseStateLocation: { + database?: string | null; + schema: string; + }; WarehouseSyncResponse: { applied: boolean; next_reconciliation_after?: null | components["schemas"]["WarehouseReconciliationCursor"]; }; + /** @description Derive progress from committed mirror rows, scoped to the active physical reader. */ + WarehouseWatermarkRead: { + /** Format: int64 */ + generation: number; + kind: components["schemas"]["WarehouseHistoryKind"]; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: uuid */ + session_id: string; + }; }; responses: never; parameters: never; @@ -2121,6 +2156,61 @@ export interface operations { }; }; }; + watermark: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WarehouseWatermarkRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": string | null; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; list_runs: { parameters: { query?: { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 56395c504e..2032927641 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -60,8 +60,8 @@ use crate::native_commands::_helpers::terminal::{ finish_lost_stdout, terminal_columns, write_output, }; use crate::native_commands::constants::{ - COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, UNCAUGHT_EXCEPTION_STATUS, - UNKNOWN_COMMAND, + COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, METADATA_RECONCILE_SECONDS, + UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; @@ -317,6 +317,7 @@ pub(crate) fn location_result( pub(crate) fn metadata_sync( (locations, generation): (Vec, i64), + reconcile_seconds: u32, ) -> LocationMetadataSync { LocationMetadataSync { locations, @@ -325,6 +326,11 @@ pub(crate) fn metadata_sync( scan_id: Uuid::now_v7(), page: 0, source_after: None, + observed_since: None, + watermark_loaded: false, + reconcile: true, + reconcile_at: Instant::now(), + reconcile_seconds, limit: MAX_HISTORY_RECORDS, pending: None, next_at: Instant::now(), @@ -344,7 +350,7 @@ pub(crate) fn history_request(state: &LocationMetadataSync) -> Result, C "source_freshness" }; serde_json::to_vec(&json!({(kind): { - "location":state.locations[state.index / 2],"after":state.source_after,"observed_since":null,"limit":state.limit + "location":state.locations[state.index / 2],"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit }})).map_err(|error| ControlError::Invalid(error.to_string())) } @@ -359,6 +365,9 @@ pub(crate) fn freeze_history_page( "reconciliation":{"scan_id":state.scan_id,"page":state.page,"state_database":location.database, "state_schema":location.schema,"day":null,"complete":page.next_after.is_none(), "after":WarehouseReconciliationCursor { observed_at: DateTime::::MAX_UTC, event_id: Uuid::nil() }}}); + if !state.reconcile { + body["reconciliation"] = serde_json::Value::Null; + } if serde_json::to_vec(&body) .map_err(|error| ControlError::Invalid(error.to_string()))? .len() @@ -374,7 +383,9 @@ pub(crate) fn freeze_history_page( )])); } } else { - body["reconciliation"]["after"] = serde_json::Value::Null; + if state.reconcile { + body["reconciliation"]["after"] = serde_json::Value::Null; + } state.pending = Some(LocationMetadataPage { body, next_source_after: page.next_after, @@ -429,9 +440,15 @@ pub(crate) fn acknowledge_metadata_page( state.index += 1; state.scan_id = Uuid::now_v7(); state.page = 0; + state.observed_since = None; + state.watermark_loaded = false; state.limit = MAX_HISTORY_RECORDS; if state.index == state.locations.len() * 2 { state.index = 0; + if state.reconcile { + state.reconcile_at = + Instant::now() + Duration::from_secs(u64::from(state.reconcile_seconds)); + } state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); drop(write_output(&[( Stream::Stderr, @@ -504,12 +521,24 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result() + .map(std::num::NonZeroU32::get) + }) + .map_err(|_| { + ControlError::Invalid( + "SQLBUILD_METADATA_RECONCILE_SECONDS must be a positive integer".into(), + ) + })?; Ok(LocationServe { project_dir, server_url: text(namespace, "location_server_url").unwrap_or_default(), location_id, target, image_id: text(namespace, "location_image_id"), + metadata_reconcile_seconds, }) } diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 4006ab3e3e..a3c0d46a69 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -93,8 +93,11 @@ pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; /// Reserve half the location work claim for acknowledgement retries after killing a callback. pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; -/// Independent read timeout and delay between complete automatic history scans. +/// Independent read timeout and delay between automatic history polls. pub(crate) const METADATA_POLL_SECONDS: u64 = 30; +/// Full key-set reconciliation is independent of the incremental poll and overlap window. +pub(crate) const METADATA_RECONCILE_SECONDS: u32 = 300; +pub(crate) const METADATA_OVERLAP_SECONDS: i64 = 300; /// A source-key cursor can repeat the last row's base64-encoded definition beside the records. pub(crate) const METADATA_REPLY_BYTES: u64 = 3 * sqlbuild_orchestration::api::constants::BODY_BYTES as u64; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 3342da615b..9b88a70aae 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -7,7 +7,9 @@ use std::time::Duration; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::platform::models::{LocationWorkClient, LocationWorkPoll}; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryPage; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseWatermarkRead, +}; use tokio::time::Instant; use crate::argv::models::Stream; @@ -17,7 +19,9 @@ use crate::native_commands::_helpers::execution::{ }; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; -use crate::native_commands::constants::{METADATA_POLL_SECONDS, METADATA_REPLY_BYTES}; +use crate::native_commands::constants::{ + METADATA_OVERLAP_SECONDS, METADATA_POLL_SECONDS, METADATA_REPLY_BYTES, +}; use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; /// One bounded source or mirror page leaves named work polling a turn between pages. @@ -35,6 +39,35 @@ pub(super) async fn sync_location_metadata( } tokio::pin!(stopped); if state.pending.is_none() { + if !state.watermark_loaded { + if state.index == 0 { + state.reconcile = Instant::now() >= state.reconcile_at; + } + if !state.reconcile { + let request = WarehouseWatermarkRead { + session_id: poll.session_id, + generation: state.generation, + location: state.locations[state.index / 2].clone(), + kind: if state.index.is_multiple_of(2) { + WarehouseHistoryKind::Fingerprints + } else { + WarehouseHistoryKind::SourceFreshness + }, + }; + let watermark = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = client.metadata_watermark(&options.target, &request) => result, + }; + let watermark = match watermark { + Ok(watermark) => watermark, + Err(error) => return retry_metadata(state, error), + }; + state.observed_since = watermark.and_then(|time| { + time.checked_sub_signed(chrono::Duration::seconds(METADATA_OVERLAP_SECONDS)) + }); + } + state.watermark_loaded = true; + } let Some(page) = read_metadata_page( (options, script), history_request(&state)?, @@ -68,25 +101,30 @@ pub(super) async fn sync_location_metadata( }; let response = match response { Ok(response) => response, - Err(error) => { - if matches!(error, ClientError::Configuration(_)) - || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) - { - return Err(ControlError::Invalid(error.to_string())); - } - state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); - drop(write_output(&[( - Stream::Stderr, - &format!( - "Warehouse history delivery failed; retaining exact page for retry: {error}\n" - ), - )])); - return Ok(Some(state)); - } + Err(error) => return retry_metadata(state, error), }; acknowledge_metadata_page(state, response).map(Some) } +fn retry_metadata( + mut state: LocationMetadataSync, + error: ClientError, +) -> Result, ControlError> { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + &format!( + "Warehouse history synchronization failed; retaining progress and exact pending page for retry: {error}\n" + ), + )])); + Ok(Some(state)) +} + /// An outer absence means shutdown; an inner absence is a failed read, never an empty table. async fn read_metadata_page( (options, script): (&LocationServe, &Path), diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 185efb131f..43e3a3f2c0 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -116,7 +116,10 @@ async fn watch_location( if published { watch.retry_at = refresh_at; published_generation = Some(generation); - metadata = Some(metadata_sync((locations, generation))); + metadata = Some(metadata_sync( + (locations, generation), + options.metadata_reconcile_seconds, + )); let Some(files) = check_published_location( options, script, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 768f978eeb..7b822402ff 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -144,6 +144,7 @@ pub(crate) struct LocationServe { pub(crate) location_id: uuid::Uuid, pub(crate) target: String, pub(crate) image_id: Option, + pub(crate) metadata_reconcile_seconds: u32, } /// Retained source observations survive compilation; warehouse exclusions never replace them. @@ -190,6 +191,11 @@ pub(crate) struct LocationMetadataSync { pub(crate) scan_id: Uuid, pub(crate) page: u32, pub(crate) source_after: Option>>, + pub(crate) observed_since: Option>, + pub(crate) watermark_loaded: bool, + pub(crate) reconcile: bool, + pub(crate) reconcile_at: tokio::time::Instant, + pub(crate) reconcile_seconds: u32, pub(crate) limit: usize, pub(crate) pending: Option, pub(crate) next_at: tokio::time::Instant, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 757556ce89..727069bc77 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -39,6 +39,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router>), + (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Reader generation fenced")), + security(("bearer" = [])))] +pub(crate) async fn watermark( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result>>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_watermark(&store, (location_id, &target), &request).await?, + )) +} + #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", operation_id = "report_source_freshness", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 7c08ea2c40..d196237820 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -49,7 +49,8 @@ use crate::api::_helpers::resources::{ trigger_reads::events, trigger_reads::ticks, warehouse_metadata::report, - warehouse_metadata::report_fingerprints + warehouse_metadata::report_fingerprints, + warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 3761245961..4787e27655 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -9,10 +9,18 @@ use crate::platform::models::{ LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, }; use crate::warehouse_metadata::models::{ - FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, + FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { + pub async fn metadata_watermark( + &self, + target: &str, + request: &WarehouseWatermarkRead, + ) -> Result>, ClientError> { + self.sync_metadata((target, "watermark"), request).await + } + pub async fn sync_fingerprints( &self, target: &str, @@ -31,11 +39,11 @@ impl LocationWorkClient { .await } - async fn sync_metadata( + async fn sync_metadata( &self, (target, table): (&str, &str), batch: &impl serde::Serialize, - ) -> Result { + ) -> Result { let endpoint = self.connection.endpoint(&[ "api", "locations", diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index fc32896be6..3590489f42 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,2 +1,3 @@ pub mod ingest_fingerprints; pub mod ingest_source_freshness; +pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs new file mode 100644 index 0000000000..c3c5480732 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs @@ -0,0 +1,42 @@ +//! The durable mirror, not a separately persisted cursor, owns incremental progress. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::Parameter; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseWatermarkRead}; + +pub async fn read_watermark( + store: &crate::store::models::ControlStore, + scope: (Uuid, &str), + request: &WarehouseWatermarkRead, +) -> Result>, ControlError> { + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let table = match request.kind { + WarehouseHistoryKind::Fingerprints => "sqb_warehouse_fingerprints", + WarehouseHistoryKind::SourceFreshness => "sqb_warehouse_freshness", + }; + let mut parameters = vec![ + Parameter::Id(scope.0), + Parameter::Text(scope.1.into()), + Parameter::Text(request.location.schema.clone()), + ]; + let database = match &request.location.database { + Some(database) => { + parameters.push(Parameter::Text(database.clone())); + "state_database=$4" + } + None => "state_database IS NULL", + }; + let (transaction, rows) = query(transaction, + &format!("SELECT MAX(observed_at) AS watermark FROM {table} WHERE location_id=$1 AND target=$2 AND state_schema=$3 \ + AND {database}"), parameters).await?; + let watermark = serde_json::from_value(rows[0]["watermark"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + commit(transaction).await?; + Ok(watermark) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index ad85cc093e..fc24e1cf66 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -35,13 +35,33 @@ pub struct SourceFreshnessBatch { } /// Physical namespace of SQLBuild's source-freshness state table, never a source query. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] #[serde(deny_unknown_fields)] pub struct WarehouseStateLocation { pub database: Option, pub schema: String, } +/// Only immutable history tables support a timestamp watermark. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseHistoryKind { + Fingerprints, + SourceFreshness, +} + +/// Derive progress from committed mirror rows, scoped to the active physical reader. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseWatermarkRead { + pub session_id: Uuid, + pub generation: i64, + pub location: WarehouseStateLocation, + pub kind: WarehouseHistoryKind, +} + /// Private interpreter input; cursor values are data and receive adapter literal rendering. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 3d25864517..090fbaf397 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -13,7 +13,7 @@ use crate::store::models::{ControlStore, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_helpers::reconstruct_warehouse_predecessor; -use crate::warehouse_metadata::models::FingerprintBatch; +use crate::warehouse_metadata::models::{FingerprintBatch, WarehouseWatermarkRead}; pub(crate) async fn fingerprint_history( store: &ControlStore, @@ -346,6 +346,139 @@ pub(crate) async fn fingerprint_history( after, "warm migration preserves imported history" ); + for (kind, database, schema, target, reader, expected) in [ + ( + "fingerprints", + None, + "main", + "development", + session_id, + Some("2026-01-01T10:00:00.000001Z"), + ), + ( + "fingerprints", + Some("analytics"), + "main", + "development", + session_id, + Some("2026-01-01T10:00:00Z"), + ), + ( + "fingerprints", + None, + "reporting", + "development", + session_id, + Some("2026-01-01T10:00:00Z"), + ), + ( + "fingerprints", + None, + "main", + "preview", + preview_session_id, + Some("2026-01-01T10:00:00Z"), + ), + ( + "source_freshness", + None, + "main", + "development", + session_id, + Some("2026-01-01T11:00:00Z"), + ), + ( + "source_freshness", + Some("analytics"), + "main", + "development", + session_id, + None, + ), + ( + "fingerprints", + None, + "missing", + "development", + session_id, + None, + ), + ] { + let request: WarehouseWatermarkRead = serde_json::from_value(json!({ + "kind": kind, "session_id": reader, "generation": 1, + "location": {"database": database, "schema": schema}, + })) + .expect("watermark request"); + assert_eq!( + serde_json::to_value( + outbound + .metadata_watermark(target, &request) + .await + .expect("durable scoped watermark") + ) + .expect("watermark"), + json!(expected) + ); + } + let watermark_url = freshness_url.replace("source-freshness", "watermark"); + let request = json!({"kind": "fingerprints", "session_id":session_id,"generation":1, + "location":{"database":null,"schema":"main"}}); + for (endpoint, mut body, status, field, value) in [ + ( + watermark_url.clone(), + request.clone(), + StatusCode::CONFLICT, + "generation", + json!(2), + ), + ( + watermark_url.clone(), + request.clone(), + StatusCode::BAD_REQUEST, + "session_id", + json!(preview_session_id), + ), + ( + watermark_url.clone(), + request.clone(), + StatusCode::UNPROCESSABLE_ENTITY, + "kind", + json!("arbitrary_table"), + ), + ( + watermark_url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string(), + ), + request.clone(), + StatusCode::FORBIDDEN, + "generation", + json!(1), + ), + ] { + body[field] = value; + assert_eq!( + client + .post(endpoint) + .bearer_auth(bearer) + .json(&body) + .send() + .await + .expect("fenced watermark") + .status(), + status + ); + } + assert_eq!( + client + .post(watermark_url) + .json(&request) + .send() + .await + .expect("anonymous watermark") + .status(), + StatusCode::UNAUTHORIZED + ); } async fn mirrored_fingerprints(store: &ControlStore) -> Vec { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 413762d597..06fdcd1877 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -98,6 +98,15 @@ def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) return adapter, connection +def append_legacy_history( + adapter: Any, connection: Any, rows: list[dict[str, str]], statements: list[str] +) -> None: + """Append synthetic arrivals with independent observation times to real history tables.""" + for row in rows: + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement.format(**row)) + + def warehouse_history_page( project: Path, table: str, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index a76820b8fa..fce5926fb7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -43,6 +43,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_contended_fingerprints, + append_legacy_history, append_retried_fingerprints, location_process, open_legacy_warehouse, @@ -547,7 +548,8 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re ], log, tmp_path / "location.stdout", - env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + env=os.environ + | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "30"}, ) as location: _ = wait_for_generation(api, "development", 1, location, log) initial_tables: dict[str, set[str]] = { @@ -606,6 +608,100 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re adapter.close(connection) +@pytest.mark.parametrize( + "test_case", + [ + AutomaticHistoryTestCase("DuckDB incremental overlap", "duckdb", "database", "legacy-run"), + AutomaticHistoryTestCase("SQLite incremental overlap", "sqlite", "path", "legacy-run"), + ], + ids=lambda case: case.description, +) +def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_preserves_history_and_empty_resyncs( + test_case: AutomaticHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json" + ).read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + append_legacy_history(adapter, connection, fixture["initial"], fixture["insert_sql"]) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files( + project_dir=project, + files={"models/orders.sql": "MODEL (description 'Orders.'); SELECT 17 AS amount"}, + ) + with snapshot_api(tmp_path / "control") as api: + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ + | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "300"}, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + for stage in fixture["stages"]: + connection = adapter.connect( + {test_case.connection_key: str(project / "warehouse.db")} + ) + arrivals: list[dict[str, str]] = stage["arrivals"] + [ + {"run_id": f"middle-{index:03}", "observed_at": "2026-02-01 11:58:00"} + for index in range(stage["overlap_count"]) + ] + append_legacy_history(adapter, connection, arrivals, fixture["insert_sql"]) + adapter.close(connection) + with closing(sqlite3.connect(control)) as database: + for table in ("sqb_warehouse_fingerprints", "sqb_warehouse_freshness"): + _ = database.execute(stage["mirror_sql"].format(table=table)) + database.commit() + expected: set[str] = set(stage["expected_runs"]) | { + f"middle-{index:03}" + for index in range(stage.get("expected_overlap_count", stage["overlap_count"])) + } + deadline: float = time.monotonic() + 45 + remaining: bool = True + while remaining: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + remaining = False + with closing(sqlite3.connect(control)) as database: + for table in ("sqb_warehouse_fingerprints", "sqb_warehouse_freshness"): + observed: set[str] = { + json.loads(record)["run_id"] + for (record,) in database.execute( + f"SELECT record_json FROM {table}" + ) + } + assert observed <= expected, (table, stage, observed) + remaining |= observed != expected + remaining |= ( + log.read_text().count("Warehouse history synchronization completed.") + < stage["completed_cycles"] + ) + time.sleep(0.05) + assert test_case.expected_run_id in observed + + @pytest.mark.parametrize( "test_case", [ From 8828854a8cdd128c846988e46ec2d93e3231426e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:53:26 +0000 Subject: [PATCH 089/138] feat: report bounded warehouse synchronization health --- .../incremental_warehouse_history.json | 11 ++- clients/typescript/src/schema.d.ts | 17 +++++ clients/typescript/tests/types.ts | 14 ++++ .../src/native_commands/_helpers/execution.rs | 40 +++++++++- .../main/_location_metadata.rs | 42 ++++++++--- .../src/platform/_helpers/work_client.rs | 15 +++- .../platform/main/report_location_health.rs | 16 ++-- .../main/report_remote_location_health.rs | 12 +-- .../src/platform/models.rs | 27 ++++++- .../src/warehouse_metadata/models.rs | 2 +- .../tests/api/warehouse_metadata/helpers.rs | 73 ++++++++++++++++++- .../compiled_project/publication/helpers.py | 51 +++++++++++-- .../publication/test_warehouse_metadata.py | 25 +++++++ 13 files changed, 303 insertions(+), 42 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json index 8bb1b8bd65..5be86e4798 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -1,14 +1,19 @@ { "initial": [{"run_id": "watermark-run", "observed_at": "2026-02-01 12:00:00"}], "stages": [ - {"arrivals": [], "overlap_count": 0, "completed_cycles": 1, "mirror_sql": "SELECT 1", "expected_runs": ["legacy-run", "watermark-run"]}, + {"arrivals": [], "overlap_count": 0, "completed_cycles": 1, "mirror_sql": "SELECT 1", "health_records_per_table": 2, "expected_runs": ["legacy-run", "watermark-run"]}, {"arrivals": [ {"run_id": "a-future", "observed_at": "2026-02-01 12:01:00"}, {"run_id": "within-overlap", "observed_at": "2026-02-01 11:58:00"}, {"run_id": "zz-boundary", "observed_at": "2026-02-01 11:55:00"}, {"run_id": "before-overlap", "observed_at": "2026-02-01 11:54:00"} - ], "overlap_count": 104, "completed_cycles": 2, "mirror_sql": "SELECT 1", "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future"]}, - {"arrivals": [], "overlap_count": 0, "completed_cycles": 3, "mirror_sql": "DELETE FROM {table}", "expected_overlap_count": 104, "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future", "before-overlap"]} + ], "overlap_count": 104, "completed_cycles": 2, "mirror_sql": "SELECT 1", "health_records_per_table": 108, "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future"]}, + {"arrivals": [], "overlap_count": 0, "completed_cycles": 3, "mirror_sql": "DELETE FROM {table}", "health_records_per_table": 110, "expected_overlap_count": 104, "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future", "before-overlap"]} + ], + "health": {"failure": null, "tables": ["fingerprints", "source_freshness"]}, + "availability": [ + {"available": false, "source": "warehouse.db", "destination": "../retained-warehouse.db", "failure": "read", "tables": ["fingerprints"]}, + {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]} ], "insert_sql": [ "INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'", diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index bd486ace2e..3bb0ebbc33 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -713,6 +713,18 @@ export interface components { healthy: boolean; /** @enum {string} */ kind: "connection"; + } | { + /** Format: int64 */ + elapsed_millis: number; + failure?: null | components["schemas"]["WarehouseSyncFailure"]; + /** Format: int64 */ + generation: number; + /** @enum {string} */ + kind: "warehouse_sync"; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: int64 */ + records_sent: number; + table: components["schemas"]["WarehouseHistoryKind"]; }; /** @description Receipt time belongs to the server; consumers derive freshness rather than storing availability. */ LocationHealthObservation: { @@ -1369,6 +1381,11 @@ export interface components { database?: string | null; schema: string; }; + /** + * @description Closed failure stages keep warehouse errors and credentials out of health observations. + * @enum {string} + */ + WarehouseSyncFailure: "watermark" | "read" | "delivery"; WarehouseSyncResponse: { applied: boolean; next_reconciliation_after?: null | components["schemas"]["WarehouseReconciliationCursor"]; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 55ff245107..5373c7d236 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -9,6 +9,20 @@ void client.GET("/api/locations/{location_id}/health", {}); // @ts-expect-error Only the native discriminated health variants are accepted. const invalid: components["schemas"]["LocationHealth"] = { kind: "unknown" }; void invalid; +const sync: components["schemas"]["LocationHealth"] = { + kind: "warehouse_sync", generation: 1, + location: { database: null, schema: "main" }, table: "fingerprints", + failure: "read", records_sent: 0, elapsed_millis: 43, +}; +void sync; +declare const observation: components["schemas"]["LocationHealthObservation"]; +if (observation.report.health.kind === "warehouse_sync") { + const records: number = observation.report.health.records_sent; + void records; +} +// @ts-expect-error Raw warehouse exceptions cannot be used as a health failure stage. +const invalidFailure: components["schemas"]["WarehouseSyncFailure"] = "warehouse exception text"; +void invalidFailure; // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 2032927641..7d6531b319 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -21,6 +21,7 @@ use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::{ LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, + WarehouseSyncFailure, }; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; @@ -32,8 +33,8 @@ use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryPage, WarehouseReconciliationCursor, WarehouseStateLocation, - WarehouseSyncResponse, + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseReconciliationCursor, + WarehouseStateLocation, WarehouseSyncResponse, }; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use tokio::process::{Child, Command}; @@ -460,6 +461,41 @@ pub(crate) fn acknowledge_metadata_page( Ok(state) } +/// Operational cost is elapsed IO time, not an estimate of warehouse billing. +pub(crate) async fn metadata_health( + state: &LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + (started, failure): (Instant, Option), +) { + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::WarehouseSync { + generation: state.generation, + location: state.locations[state.index / 2].clone(), + table: if state.index.is_multiple_of(2) { + WarehouseHistoryKind::Fingerprints + } else { + WarehouseHistoryKind::SourceFreshness + }, + failure, + records_sent: state + .pending + .as_ref() + .and_then(|page| page.body["records"].as_array()) + .map_or(0, Vec::len) as u64, + elapsed_millis: started.elapsed().as_millis() as u64, + }, + }; + if let Err(error) = client.report_health(&report).await { + drop(write_output(&[( + Stream::Stderr, + &format!("Warehouse synchronization health report failed: {error}\n"), + )])); + } +} + /// Health IO remains native even while a user-code child holds its interpreter's GIL. pub(crate) async fn report_location_health( options: &LocationServe, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 9b88a70aae..f2dfaeaeed 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -5,7 +5,9 @@ use std::path::Path; use std::time::Duration; use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::platform::models::{LocationWorkClient, LocationWorkPoll}; +use sqlbuild_orchestration::platform::models::{ + LocationWorkClient, LocationWorkPoll, WarehouseSyncFailure, +}; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryKind, WarehouseHistoryPage, WarehouseWatermarkRead, @@ -15,7 +17,7 @@ use tokio::time::Instant; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ acknowledge_metadata_page, deliver_metadata_page, freeze_history_page, history_request, - location_result, + location_result, metadata_health, }; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; @@ -37,6 +39,7 @@ pub(super) async fn sync_location_metadata( { return Ok(Some(state)); } + let started = Instant::now(); tokio::pin!(stopped); if state.pending.is_none() { if !state.watermark_loaded { @@ -60,7 +63,13 @@ pub(super) async fn sync_location_metadata( }; let watermark = match watermark { Ok(watermark) => watermark, - Err(error) => return retry_metadata(state, error), + Err(error) => { + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Watermark))) => {}, + } + return retry_metadata(state, error); + } }; state.observed_since = watermark.and_then(|time| { time.checked_sub_signed(chrono::Duration::seconds(METADATA_OVERLAP_SECONDS)) @@ -68,16 +77,15 @@ pub(super) async fn sync_location_metadata( } state.watermark_loaded = true; } - let Some(page) = read_metadata_page( - (options, script), - history_request(&state)?, - stopped.as_mut(), - ) - .await? + let Some(page) = read_metadata_page((options, script), &state, stopped.as_mut()).await? else { return Ok(None); }; let Some(page) = page else { + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Read))) => {}, + } state.limit = (state.limit / 2).max(1); state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); drop(write_output(&[( @@ -101,8 +109,18 @@ pub(super) async fn sync_location_metadata( }; let response = match response { Ok(response) => response, - Err(error) => return retry_metadata(state, error), + Err(error) => { + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Delivery))) => {}, + } + return retry_metadata(state, error); + } }; + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, None)) => {}, + } acknowledge_metadata_page(state, response).map(Some) } @@ -128,12 +146,12 @@ fn retry_metadata( /// An outer absence means shutdown; an inner absence is a failed read, never an empty table. async fn read_metadata_page( (options, script): (&LocationServe, &Path), - request: Vec, + state: &LocationMetadataSync, stopped: impl Future>, ) -> Result>, ControlError> { let (mut child, reply) = match location_result::>( (options, script), - (request, METADATA_REPLY_BYTES), + (history_request(state)?, METADATA_REPLY_BYTES), ) { Ok(exchange) => exchange, Err(_) => return Ok(Some(None)), diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 4787e27655..cc6f2ed4aa 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -6,13 +6,26 @@ use crate::api::errors::ClientError; use crate::api::main::client_response::client_response; use crate::api::models::MutationResponse; use crate::platform::models::{ - LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, + LocationHealthReport, LocationWorkClaim, LocationWorkClient, LocationWorkPoll, + LocationWorkResult, }; use crate::warehouse_metadata::models::{ FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { + pub async fn report_health(&self, report: &LocationHealthReport) -> Result { + let endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "health", + ])?; + let response: MutationResponse = + client_response(self.connection.client.post(endpoint).json(report)).await?; + Ok(response.applied) + } + pub async fn metadata_watermark( &self, target: &str, diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs index 389d80fa18..7d2dad4bc8 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -25,7 +25,8 @@ pub async fn report_location_health( let generation = match report.health { LocationHealth::Heartbeat {} => None, LocationHealth::CompileFailed { generation } - | LocationHealth::Connection { generation, .. } => Some(generation), + | LocationHealth::Connection { generation, .. } + | LocationHealth::WarehouseSync { generation, .. } => Some(generation), }; if generation.is_some_and(|generation| generation < 1) { return Err(ControlError::Invalid( @@ -65,16 +66,21 @@ pub async fn report_location_health( if location.is_empty() { return Err(ControlError::Invalid("active location is required".into())); } - let transaction = if let LocationHealth::Connection { generation, .. } = report.health { + let transaction = if let LocationHealth::Connection { generation, .. } + | LocationHealth::WarehouseSync { generation, .. } = report.health + { let (transaction, snapshot) = query( transaction, "SELECT snapshot_id FROM sqb_snapshots WHERE location_id=$1 AND target=$2 AND generation=$3", vec![Parameter::Id(location_id), Parameter::Text(report.target.clone()), Parameter::Integer(generation)], ).await?; if snapshot.is_empty() { - return Err(ControlError::Invalid( - "connection health requires a published generation".into(), - )); + let message = if matches!(report.health, LocationHealth::WarehouseSync { .. }) { + "warehouse sync health requires a published generation" + } else { + "connection health requires a published generation" + }; + return Err(ControlError::Invalid(message.into())); } transaction } else { diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs index 5963cf20b6..ac238b8232 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs @@ -3,9 +3,7 @@ use uuid::Uuid; use crate::api::errors::ClientError; -use crate::api::main::client_response::client_response; -use crate::api::main::connect_client::connect_client; -use crate::api::models::MutationResponse; +use crate::platform::main::connect_location_work::connect_location_work; use crate::platform::models::LocationHealthReport; pub async fn report_remote_location_health( @@ -13,9 +11,7 @@ pub async fn report_remote_location_health( location_id: Uuid, report: &LocationHealthReport, ) -> Result { - let connection = connect_client(server_url, bearer)?; - let url = connection.endpoint(&["api", "locations", &location_id.to_string(), "health"])?; - let response: MutationResponse = - client_response(connection.client.post(url).json(report)).await?; - Ok(response.applied) + connect_location_work((server_url, bearer), location_id)? + .report_health(report) + .await } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index fab9c20eab..fc5bf7a971 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseStateLocation}; /// One pooled outbound connection; the location never receives a store handle. #[derive(Clone, Debug)] @@ -21,8 +22,30 @@ pub struct LocationWorkClient { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationHealth { Heartbeat {}, - CompileFailed { generation: i64 }, - Connection { generation: i64, healthy: bool }, + CompileFailed { + generation: i64, + }, + Connection { + generation: i64, + healthy: bool, + }, + WarehouseSync { + generation: i64, + location: WarehouseStateLocation, + table: WarehouseHistoryKind, + failure: Option, + records_sent: u64, + elapsed_millis: u64, + }, +} + +/// Closed failure stages keep warehouse errors and credentials out of health observations. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseSyncFailure { + Watermark, + Read, + Delivery, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index fc24e1cf66..91a98c517f 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -45,7 +45,7 @@ pub struct WarehouseStateLocation { } /// Only immutable history tables support a timestamp watermark. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum WarehouseHistoryKind { Fingerprints, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 090fbaf397..4da2ffc4b0 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -4,7 +4,7 @@ use reqwest::{Client, StatusCode}; use serde_json::{Value, json}; use uuid::Uuid; -use crate::platform::models::LocationWorkClient; +use crate::platform::models::{LocationHealthReport, LocationWorkClient}; use crate::store::main::decompress::decompress; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, @@ -479,6 +479,77 @@ pub(crate) async fn fingerprint_history( .status(), StatusCode::UNAUTHORIZED ); + let health_url = format!( + "{}/health", + freshness_url + .split("/targets/") + .next() + .expect("location URL") + ); + for failure in [None, Some("watermark"), Some("read"), Some("delivery")] { + let report: LocationHealthReport = serde_json::from_value(json!({ + "event_id":Uuid::now_v7(), "session_id":session_id, "target":"development", + "health":{"kind":"warehouse_sync", "generation":1, + "location":{"database":null,"schema":"main"},"table":"fingerprints", + "failure":failure,"records_sent":17,"elapsed_millis":43}, + })) + .expect("bounded sync health"); + for applied in [true, false] { + assert_eq!( + outbound + .report_health(&report) + .await + .expect("report health"), + applied + ); + } + let observations: Vec = client + .get(&health_url) + .bearer_auth(bearer) + .send() + .await + .expect("health read") + .error_for_status() + .expect("authorized health") + .json() + .await + .expect("health observations"); + assert_eq!( + observations + .iter() + .filter(|row| row["report"] == json!(report)) + .count(), + 1, + "exact retries preserve one historical health observation" + ); + let mut invalid = json!(report); + invalid["event_id"] = json!(Uuid::now_v7()); + invalid["health"]["generation"] = json!(2); + assert_eq!( + client + .post(&health_url) + .bearer_auth(bearer) + .json(&invalid) + .send() + .await + .expect("unpublished sync health") + .status(), + StatusCode::BAD_REQUEST + ); + invalid["health"]["generation"] = json!(1); + invalid["health"]["failure"] = json!("unbounded exception text"); + assert_eq!( + client + .post(&health_url) + .bearer_auth(bearer) + .json(&invalid) + .send() + .await + .expect("closed failure vocabulary") + .status(), + StatusCode::UNPROCESSABLE_ENTITY + ); + } } async fn mirrored_fingerprints(store: &ControlStore) -> Vec { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 06fdcd1877..a516ba325a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -188,6 +188,49 @@ def wait_for_log( time.sleep(0.02) +def location_health_reports(api: SnapshotApi) -> list[dict[str, Any]]: + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=2) as response: + return json.load(response) + + +def wait_for_warehouse_sync_health( + api: SnapshotApi, + expected: dict[str, Any], + process: subprocess.Popen[bytes], + log: Path, + *, + after: datetime, +) -> list[dict[str, Any]]: + deadline: float = time.monotonic() + 45 + reports: list[dict[str, Any]] = [] + while {report["report"]["health"]["table"] for report in reports} != set(expected["tables"]): + assert process.poll() is None, log.read_text() + by_outcome: dict[tuple[str, str | None, bool], list[dict[str, Any]]] = {} + for report in location_health_reports(api): + health: dict[str, Any] = report["report"]["health"] + key: tuple[str, str | None, bool] = ( + health["kind"], + health.get("failure"), + datetime.fromisoformat(report["received_at"]) > after, + ) + by_outcome.setdefault(key, []).append(report) + reports = by_outcome.get(("warehouse_sync", expected["failure"], True), []) + assert time.monotonic() < deadline, (reports, log.read_text()) + time.sleep(0.05) + for report in reports: + assert report["report"]["target"] == "development" + health = report["report"]["health"] + assert health["generation"] == 1 + assert health["location"] == {"database": None, "schema": "main"} + assert 0 <= health["records_sent"] <= 100 + assert health["elapsed_millis"] > 0 + return reports + + def wait_for_location_health( api: SnapshotApi, target: str, @@ -198,18 +241,12 @@ def wait_for_location_health( minimum_reports: int = 1, ) -> dict[str, Any]: deadline: float = time.monotonic() + 15 + 30 * (minimum_reports - 1) - request: Request = Request( - f"{api.url}/api/locations/{api.location_id}/health", - headers={"Authorization": f"Bearer {api.bearer}"}, - ) key: tuple[str, str] = (target, json.dumps(health, sort_keys=True)) observations: dict[tuple[str, str], dict[str, dict[str, Any]]] = {} while len(observations.get(key, ())) < minimum_reports: assert process.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() - with urlopen(request, timeout=2) as response: - reports: list[dict[str, Any]] = json.load(response) - for report in reversed(reports): + for report in reversed(location_health_reports(api)): event_id: str = report["report"]["event_id"] observed_key: tuple[str, str] = ( report["report"]["target"], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index fce5926fb7..b0a734a935 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -50,6 +50,7 @@ snapshot_api, sync_warehouse_history, wait_for_generation, + wait_for_warehouse_sync_health, warehouse_history_page, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -660,6 +661,7 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "300"}, ) as location: _ = wait_for_generation(api, "development", 1, location, log) + observed_health_at: datetime = datetime(1970, 1, 1, tzinfo=UTC) for stage in fixture["stages"]: connection = adapter.connect( {test_case.connection_key: str(project / "warehouse.db")} @@ -700,6 +702,29 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre ) time.sleep(0.05) assert test_case.expected_run_id in observed + reports: list[dict[str, Any]] = wait_for_warehouse_sync_health( + api, fixture["health"], location, log, after=observed_health_at + ) + sent_counts: dict[str, int] = dict.fromkeys(fixture["health"]["tables"], 0) + for report in reports: + sent_counts[report["report"]["health"]["table"]] += report["report"]["health"][ + "records_sent" + ] + assert sent_counts == dict.fromkeys( + fixture["health"]["tables"], stage["health_records_per_table"] + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + for availability in fixture["availability"]: + _ = (project / availability["source"]).rename(project / availability["destination"]) + reports = wait_for_warehouse_sync_health( + api, availability, location, log, after=observed_health_at + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + assert (project / "warehouse.db").exists() == availability["available"] @pytest.mark.parametrize( From b586209ec5cc57d1d68ce222f830e197a4936cde Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:38:34 +0000 Subject: [PATCH 090/138] feat: synchronize warehouse audit-result history --- .../orchestration/audit_result_history.json | 51 ++++ clients/typescript/src/schema.d.ts | 125 +++++++++- clients/typescript/tests/types.ts | 8 + .../src/native_commands/_helpers/execution.rs | 66 ++--- .../main/_location_metadata.rs | 12 +- .../src/native_commands/models.rs | 14 +- .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 21 +- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 11 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/warehouse_audit_results.sql | 14 ++ .../main/ingest_audit_results.rs | 86 +++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../warehouse_metadata/main/read_watermark.rs | 7 +- .../src/warehouse_metadata/models.rs | 99 +++++++- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_warehouse_metadata.rs | 7 + .../audit_results/helpers.rs | 231 ++++++++++++++++++ .../tests/credentials/test_audit.rs | 2 + .../tests/store/helpers.rs | 2 + .../_helpers/location_command/runtime.rs | 9 +- .../native_host/warehouse_metadata.rs | 104 ++++++-- .../src/state_tables/main/read_history_sql.rs | 3 +- .../sqlbuild-state/src/state_tables/models.rs | 2 + .../compiled_project/publication/helpers.py | 56 ++++- .../publication/test_warehouse_metadata.py | 153 ++++++++++++ 28 files changed, 1025 insertions(+), 72 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_audit_results.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json b/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json new file mode 100644 index 0000000000..acab1bf063 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json @@ -0,0 +1,51 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "result_id": "result-000", + "schema_version": 1, + "occurred_at": "2026-02-03T04:05:06Z", + "invocation_id": "invocation-1", + "run_id": "", + "audit_name": "valid_orders", + "audit_definition_name": "dq_column_rate", + "audit_description": "Valid order percentage", + "binding_key": "orders.valid_orders", + "definition_fingerprint": "definition-fingerprint", + "execution_fingerprint": "execution-fingerprint", + "evaluation_mode": "measurement", + "run_scope_phase": "final", + "attachment_kind": "factory", + "attached_target_kind": "model", + "attached_target_name": "orders", + "attached_column_name": "order_id", + "target_database": null, + "target_schema": "main", + "target_name": "orders_physical", + "severity": "warn", + "outcome": "pass", + "execution_error": null, + "violation_count": null, + "measured_value": "99.5", + "sample_count": 200, + "sample_unit": "rows", + "minimum_samples": 100, + "thresholds_json": "{\"warn\":{\"below\":99}}", + "evidence_json": "[{\"order_id\":\"o'rders 🧾\"}]", + "evidence_count": 1, + "evidence_truncated": "true", + "evidence_error": null, + "measurement_sql": "SELECT 99.5 AS valid_rate", + "evidence_sql": null, + "executed_sql": "SELECT 99.5 AS valid_rate", + "sql_digest": "digest", + "metadata_json": "{\"owner\":\"data\"}", + "reused": "false", + "audit_meta_json": "{\"owner\":\"orders\"}" + }, + "source_rows": 104, + "duplicate_rows": 105, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "result-103", + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 3bb0ebbc33..9bee33e5bf 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -275,6 +275,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/audit-results": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_audit_results"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/fingerprints": { parameters: { query?: never; @@ -591,6 +607,14 @@ export interface components { ApiErrorBody: { error: string; }; + AuditResultsBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseAuditRecord"][]; + /** Format: uuid */ + session_id: string; + }; AuthenticationDenial: { /** Format: uuid */ decision_id: string; @@ -1316,6 +1340,57 @@ export interface components { }; /** @enum {string} */ TriggerStatus: "running" | "paused"; + /** @description Preserve the warehouse representation, including text-encoded measurements and booleans. */ + WarehouseAuditRecord: { + attached_column_name?: string | null; + attached_target_kind?: string | null; + attached_target_name?: string | null; + attachment_kind: string; + audit_definition_name: string; + audit_description?: string | null; + audit_meta_json?: string | null; + audit_name: string; + binding_key: string; + definition_fingerprint: string; + evaluation_mode: string; + /** Format: int64 */ + evidence_count?: number | null; + evidence_error?: string | null; + evidence_json?: string | null; + evidence_sql?: string | null; + evidence_truncated?: string | null; + executed_sql?: string | null; + execution_error?: string | null; + execution_fingerprint: string; + invocation_id: string; + measured_value?: string | null; + measurement_sql?: string | null; + metadata_json?: string | null; + /** Format: int64 */ + minimum_samples?: number | null; + /** Format: date-time */ + occurred_at: string; + outcome: string; + result_id: string; + reused: string; + run_id: string; + run_scope_phase: string; + /** Format: int64 */ + sample_count?: number | null; + sample_unit?: string | null; + /** Format: int64 */ + schema_version: number; + severity: string; + sql_digest?: string | null; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + thresholds_json?: string | null; + /** Format: int64 */ + violation_count?: number | null; + }; /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ WarehouseFingerprintRecord: { definition: string; @@ -1356,7 +1431,7 @@ export interface components { * @description Only immutable history tables support a timestamp watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results"; WarehouseReconciliationCursor: { /** Format: uuid */ event_id: string; @@ -2077,6 +2152,54 @@ export interface operations { }; }; }; + report_audit_results: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AuditResultsBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_fingerprints: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 5373c7d236..10121c006c 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -23,6 +23,14 @@ if (observation.report.health.kind === "warehouse_sync") { // @ts-expect-error Raw warehouse exceptions cannot be used as a health failure stage. const invalidFailure: components["schemas"]["WarehouseSyncFailure"] = "warehouse exception text"; void invalidFailure; +const auditHistory: components["schemas"]["WarehouseHistoryKind"] = "audit_results"; +const auditMeasurement: components["schemas"]["WarehouseAuditRecord"]["measured_value"] = "99.5"; +void auditHistory; +void auditMeasurement; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/audit-results", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 7d6531b319..cd862f286d 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -345,13 +345,9 @@ pub(crate) fn history_request(state: &LocationMetadataSync) -> Result, C "Reading warehouse history for automatic synchronization...\n", )])); } - let kind = if state.index.is_multiple_of(2) { - "fingerprints" - } else { - "source_freshness" - }; + let kind = state.kind().as_str(); serde_json::to_vec(&json!({(kind): { - "location":state.locations[state.index / 2],"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit + "location":state.location(),"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit }})).map_err(|error| ControlError::Invalid(error.to_string())) } @@ -361,7 +357,7 @@ pub(crate) fn freeze_history_page( page: WarehouseHistoryPage, poll: &LocationWorkPoll, ) -> Result { - let location = &state.locations[state.index / 2]; + let location = state.location(); let mut body = json!({"session_id":poll.session_id,"generation":state.generation,"records":page.records, "reconciliation":{"scan_id":state.scan_id,"page":state.page,"state_database":location.database, "state_schema":location.schema,"day":null,"complete":page.next_after.is_none(), @@ -398,24 +394,36 @@ pub(crate) fn freeze_history_page( pub(crate) async fn deliver_metadata_page( (client, target): (&LocationWorkClient, &str), body: &serde_json::Value, - fingerprints: bool, + kind: WarehouseHistoryKind, ) -> Result { - if fingerprints { - client - .sync_fingerprints( - target, - &serde_json::from_value(body.clone()) - .map_err(|_| ClientError::Configuration("invalid fingerprint batch"))?, - ) - .await - } else { - client - .sync_source_freshness( - target, - &serde_json::from_value(body.clone()) - .map_err(|_| ClientError::Configuration("invalid freshness batch"))?, - ) - .await + match kind { + WarehouseHistoryKind::Fingerprints => { + client + .sync_fingerprints( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid fingerprint batch"))?, + ) + .await + } + WarehouseHistoryKind::SourceFreshness => { + client + .sync_source_freshness( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid freshness batch"))?, + ) + .await + } + WarehouseHistoryKind::AuditResults => { + client + .sync_audit_results( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid audit results batch"))?, + ) + .await + } } } @@ -444,7 +452,7 @@ pub(crate) fn acknowledge_metadata_page( state.observed_since = None; state.watermark_loaded = false; state.limit = MAX_HISTORY_RECORDS; - if state.index == state.locations.len() * 2 { + if state.index == state.locations.len() * WarehouseHistoryKind::ALL.len() { state.index = 0; if state.reconcile { state.reconcile_at = @@ -473,12 +481,8 @@ pub(crate) async fn metadata_health( target: poll.target.clone(), health: LocationHealth::WarehouseSync { generation: state.generation, - location: state.locations[state.index / 2].clone(), - table: if state.index.is_multiple_of(2) { - WarehouseHistoryKind::Fingerprints - } else { - WarehouseHistoryKind::SourceFreshness - }, + location: state.location().clone(), + table: state.kind(), failure, records_sent: state .pending diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index f2dfaeaeed..f93388caeb 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -10,7 +10,7 @@ use sqlbuild_orchestration::platform::models::{ }; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryKind, WarehouseHistoryPage, WarehouseWatermarkRead, + WarehouseHistoryPage, WarehouseWatermarkRead, }; use tokio::time::Instant; @@ -50,12 +50,8 @@ pub(super) async fn sync_location_metadata( let request = WarehouseWatermarkRead { session_id: poll.session_id, generation: state.generation, - location: state.locations[state.index / 2].clone(), - kind: if state.index.is_multiple_of(2) { - WarehouseHistoryKind::Fingerprints - } else { - WarehouseHistoryKind::SourceFreshness - }, + location: state.location().clone(), + kind: state.kind(), }; let watermark = tokio::select! { result = &mut stopped => { result?; return Ok(None); }, @@ -105,7 +101,7 @@ pub(super) async fn sync_location_metadata( }; let response = tokio::select! { result = &mut stopped => { result?; return Ok(None); }, - response = deliver_metadata_page((client, &options.target), &pending.body, state.index.is_multiple_of(2)) => response, + response = deliver_metadata_page((client, &options.target), &pending.body, state.kind()) => response, }; let response = match response { Ok(response) => response, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 7b822402ff..cd12aba21e 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -8,7 +8,9 @@ use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; -use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseStateLocation, +}; use uuid::Uuid; use crate::argv::models::Namespace; @@ -201,6 +203,16 @@ pub(crate) struct LocationMetadataSync { pub(crate) next_at: tokio::time::Instant, } +impl LocationMetadataSync { + pub(crate) fn kind(&self) -> WarehouseHistoryKind { + WarehouseHistoryKind::ALL[self.index % WarehouseHistoryKind::ALL.len()] + } + + pub(crate) fn location(&self) -> &WarehouseStateLocation { + &self.locations[self.index / WarehouseHistoryKind::ALL.len()] + } +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 727069bc77..c4090b0fd6 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -39,6 +39,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_audit_results(&store, (location_id, &target), &batch).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index d196237820..8a007c72d7 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -50,6 +50,7 @@ use crate::api::_helpers::resources::{ trigger_reads::ticks, warehouse_metadata::report, warehouse_metadata::report_fingerprints, + warehouse_metadata::report_audit_results, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index cc6f2ed4aa..0f0cd7acbf 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,7 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, + AuditResultsBatch, FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, + WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -42,6 +43,14 @@ impl LocationWorkClient { self.sync_metadata((target, "fingerprints"), batch).await } + pub async fn sync_audit_results( + &self, + target: &str, + batch: &AuditResultsBatch, + ) -> Result { + self.sync_metadata((target, "audit-results"), batch).await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 2068c2d4ff..49bbeba5ca 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -27,6 +27,7 @@ pub(crate) const ADMINISTRATOR_INITIALIZATION: &str = pub(crate) const EXTERNAL_SESSIONS: &str = include_str!("../external_sessions.sql"); pub(crate) const WAREHOUSE_RECONCILIATION: &str = include_str!("../warehouse_reconciliation.sql"); pub(crate) const AUDIT_STORAGE_ORDER: &str = include_str!("../audit_storage_order.sql"); +pub(crate) const WAREHOUSE_AUDIT_RESULTS: &str = include_str!("../warehouse_audit_results.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index b13dfa4120..282fe89ca7 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -7,8 +7,8 @@ use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, - WAREHOUSE_RECONCILIATION, render, verify_migrations, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, + WAREHOUSE_FRESHNESS, WAREHOUSE_RECONCILIATION, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -51,6 +51,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("external_sessions", EXTERNAL_SESSIONS), ("warehouse_reconciliation", WAREHOUSE_RECONCILIATION), ("audit_storage_order", AUDIT_STORAGE_ORDER), + ("warehouse_audit_results", WAREHOUSE_AUDIT_RESULTS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_audit_results.sql b/crates/sqlbuild-orchestration/src/store/warehouse_audit_results.sql new file mode 100644 index 0000000000..d5506e26f2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_audit_results.sql @@ -0,0 +1,14 @@ +CREATE TABLE sqb_warehouse_audit_results ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + audit_name TEXT NOT NULL, + outcome TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_audit_results_reconciliation ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs new file mode 100644 index 0000000000..e0142e7b73 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs @@ -0,0 +1,86 @@ +//! Import warehouse-owned check results without deriving scheduler execution or recovery. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, scoped_event_id, validate_fields, +}; +use crate::warehouse_metadata::_helpers::reconciliation::begin_reconciliation; +use crate::warehouse_metadata::models::{AuditResultsBatch, WarehouseSyncResponse}; + +pub async fn ingest_audit_results( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &AuditResultsBatch, +) -> Result { + validate_fields( + ("audit results", batch.generation, batch.records.len()), + batch + .records + .iter() + .flat_map(|record| [&record.result_id, &record.state_schema, &record.audit_name]) + .any(|value| value.trim().is_empty()), + batch.reconciliation.as_ref().is_some_and(|page| { + batch.records.iter().any(|record| { + record.state_database != page.state_database + || record.state_schema != page.state_schema + || page + .day + .is_some_and(|day| record.occurred_at.date_naive() != day) + }) + }), + )?; + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await?; + if let Some(page) = &batch.reconciliation { + transaction = begin_reconciliation( + transaction, + ("sqb_warehouse_audit_results", location_id, target), + page, + json!(batch), + ) + .await?; + } + let mut applied = false; + let mut keys: Vec = Vec::new(); + for record in &batch.records { + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_audit_results", + ), + &record.result_id, + )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } + let now = transaction.now; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_audit_results (event_id,location_id,target,source_event_id,observed_at,state_database,state_schema,audit_name,outcome,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(record.result_id.clone()), Parameter::Time(record.occurred_at), + Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Text(record.audit_name.clone()), Parameter::Text(record.outcome.clone()), + Parameter::Json(json!(record)), Parameter::Time(now)]).await?; + applied |= !inserted.is_empty(); + } + commit_ingestion( + transaction, + ("sqb_warehouse_audit_results", location_id, target), + (batch.reconciliation.as_ref(), &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 3590489f42..31a249d862 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,3 +1,4 @@ +pub mod ingest_audit_results; pub mod ingest_fingerprints; pub mod ingest_source_freshness; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs index c3c5480732..6f0420bbdd 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs @@ -7,7 +7,7 @@ use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; use crate::store::models::Parameter; use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; -use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseWatermarkRead}; +use crate::warehouse_metadata::models::WarehouseWatermarkRead; pub async fn read_watermark( store: &crate::store::models::ControlStore, @@ -16,10 +16,7 @@ pub async fn read_watermark( ) -> Result>, ControlError> { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; - let table = match request.kind { - WarehouseHistoryKind::Fingerprints => "sqb_warehouse_fingerprints", - WarehouseHistoryKind::SourceFreshness => "sqb_warehouse_freshness", - }; + let table = request.kind.mirror_table(); let mut parameters = vec![ Parameter::Id(scope.0), Parameter::Text(scope.1.into()), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 91a98c517f..610d9cfc47 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -34,6 +34,64 @@ pub struct SourceFreshnessBatch { pub reconciliation: Option, } +/// Preserve the warehouse representation, including text-encoded measurements and booleans. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseAuditRecord { + pub state_database: Option, + pub state_schema: String, + pub result_id: String, + pub schema_version: i64, + pub occurred_at: DateTime, + pub invocation_id: String, + pub run_id: String, + pub audit_name: String, + pub audit_definition_name: String, + pub audit_description: Option, + pub binding_key: String, + pub definition_fingerprint: String, + pub execution_fingerprint: String, + pub evaluation_mode: String, + pub run_scope_phase: String, + pub attachment_kind: String, + pub attached_target_kind: Option, + pub attached_target_name: Option, + pub attached_column_name: Option, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub severity: String, + pub outcome: String, + pub execution_error: Option, + pub violation_count: Option, + pub measured_value: Option, + pub sample_count: Option, + pub sample_unit: Option, + pub minimum_samples: Option, + pub thresholds_json: Option, + pub evidence_json: Option, + pub evidence_count: Option, + pub evidence_truncated: Option, + pub evidence_error: Option, + pub measurement_sql: Option, + pub evidence_sql: Option, + pub executed_sql: Option, + pub sql_digest: Option, + pub metadata_json: Option, + pub reused: String, + pub audit_meta_json: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct AuditResultsBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + /// Physical namespace of SQLBuild's source-freshness state table, never a source query. #[derive( Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, @@ -50,6 +108,33 @@ pub struct WarehouseStateLocation { pub enum WarehouseHistoryKind { Fingerprints, SourceFreshness, + AuditResults, +} + +impl WarehouseHistoryKind { + pub const ALL: [Self; 3] = [ + Self::Fingerprints, + Self::SourceFreshness, + Self::AuditResults, + ]; + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Fingerprints => "fingerprints", + Self::SourceFreshness => "source_freshness", + Self::AuditResults => "audit_results", + } + } + + #[must_use] + pub const fn mirror_table(self) -> &'static str { + match self { + Self::Fingerprints => "sqb_warehouse_fingerprints", + Self::SourceFreshness => "sqb_warehouse_freshness", + Self::AuditResults => "sqb_warehouse_audit_results", + } + } } /// Derive progress from committed mirror rows, scoped to the active physical reader. @@ -142,12 +227,23 @@ pub struct FingerprintRead { pub limit: Option, } -/// The private interpreter accepts only these two known state tables, never arbitrary SQL. +/// Duplicate result IDs may be appended; timestamps distinguish their ordered observations. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuditResultsRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 3]>, + pub observed_since: Option>, + pub limit: Option, +} + +/// The private interpreter accepts only known state tables, never arbitrary SQL. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum WarehouseHistoryRead { SourceFreshness(SourceFreshnessRead), Fingerprints(FingerprintRead), + AuditResults(AuditResultsRead), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -155,6 +251,7 @@ pub enum WarehouseHistoryRead { pub enum WarehouseHistoryRecord { SourceFreshness(WarehouseFreshnessRecord), Fingerprint(WarehouseFingerprintRecord), + AuditResult(Box), } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 359a2ce7e2..c303668478 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -38,6 +38,8 @@ mod test_types; mod test_warehouse_metadata; #[path = "test_worker.rs"] mod test_worker; +#[path = "warehouse_metadata/audit_results/helpers.rs"] +mod warehouse_audit_results_helpers; #[path = "warehouse_metadata/helpers.rs"] mod warehouse_metadata_helpers; #[path = "warehouse_metadata/reconciliation/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 3079949a9e..7f2accc76c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -322,6 +322,13 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit (session_id, preview_session), ) .await; + crate::api::tests::warehouse_audit_results_helpers::audit_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs new file mode 100644 index 0000000000..a39992e181 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs @@ -0,0 +1,231 @@ +//! External audit facts retain their identities, physical scopes and atomic replay semantics. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::LocationWorkClient; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::models::{AuditResultsBatch, WarehouseWatermarkRead}; + +pub(crate) async fn audit_history( + store: &ControlStore, + outbound: &LocationWorkClient, + (session_id, preview_session): (Uuid, Uuid), + (client, freshness_url, bearer): (&Client, &str, &str), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json"); + let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("audit history"); + let record = fixture["record"].clone(); + let batch = json!({"session_id": session_id, "generation": 1, "records": [record, record]}); + let typed: AuditResultsBatch = serde_json::from_value(batch.clone()).expect("audit batch"); + for expected in [true, false] { + assert_eq!( + outbound + .sync_audit_results("development", &typed) + .await + .expect("sync") + .applied, + expected + ); + } + let before = mirrored_audits(store).await; + assert_eq!( + before.len(), + 1, + "duplicate deterministic result IDs deduplicate" + ); + assert_eq!( + before[0]["record_json"], record, + "empty external run ID and warehouse text survive" + ); + let url = freshness_url.replace("source-freshness", "audit-results"); + for (field, value, status) in [ + ("generation", json!(2), StatusCode::CONFLICT), + ( + "session_id", + json!(preview_session), + StatusCode::BAD_REQUEST, + ), + ( + "records", + json!(vec![record.clone(); 101]), + StatusCode::BAD_REQUEST, + ), + ] { + let mut invalid = batch.clone(); + invalid[field] = value; + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&invalid) + .send() + .await + .expect("fenced batch") + .status(), + status + ); + } + assert_eq!( + client + .post(url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string() + )) + .bearer_auth(bearer) + .json(&batch) + .send() + .await + .expect("other location") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(&url) + .json(&batch) + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let mut scoped = record.clone(); + scoped["state_schema"] = json!("reporting"); + let mut database = record.clone(); + database["state_database"] = json!("analytics"); + let scopes: AuditResultsBatch = serde_json::from_value(json!({"session_id":session_id, + "generation":1, "records":[scoped,database]})) + .expect("physical scopes"); + assert!( + outbound + .sync_audit_results("development", &scopes) + .await + .expect("scope") + .applied + ); + let preview: AuditResultsBatch = serde_json::from_value(json!({"session_id":preview_session, + "generation":1, "records":[record]})) + .expect("preview"); + assert!( + outbound + .sync_audit_results("preview", &preview) + .await + .expect("target scope") + .applied + ); + assert_eq!(mirrored_audits(store).await.len(), 4); + let watermark: WarehouseWatermarkRead = serde_json::from_value(json!({"session_id":session_id, + "generation":1,"kind":"audit_results","location":{"database":null,"schema":"main"}})) + .expect("watermark"); + assert_eq!( + json!( + outbound + .metadata_watermark("development", &watermark) + .await + .expect("watermark") + ), + record["occurred_at"] + ); + let fault = TransactionFault { + description: "audit page rolls back atomically", + table: "sqb_warehouse_audit_results", + operation: "INSERT", + condition: "WHEN NEW.source_event_id='rejected-result'", + sqlite_body: "SELECT RAISE(ABORT, 'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + let before = mirrored_audits(store).await; + let mut first = record.clone(); + first["result_id"] = json!("accepted-result"); + let mut second = record.clone(); + second["result_id"] = json!("rejected-result"); + let additional = json!({"session_id":session_id,"generation":1,"records":[first,second]}); + install_fault(store, &fault).await; + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("fault") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + assert_eq!( + mirrored_audits(store).await, + before, + "no partial page survives" + ); + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("write"), + "DELETE FROM sqb_warehouse_audit_results", + vec![], + ) + .await + .expect("empty mirror"); + commit(transaction).await.expect("wipe"); + assert!( + outbound + .sync_audit_results("development", &typed) + .await + .expect("rebuild") + .applied + ); + assert!( + outbound + .sync_audit_results("development", &scopes) + .await + .expect("rebuild scopes") + .applied + ); + assert!( + outbound + .sync_audit_results("preview", &preview) + .await + .expect("rebuild target") + .applied + ); + assert_eq!( + mirrored_audits(store).await, + before, + "identical facts after rebuild" + ); + migrate_store(store).await.expect("warm migration"); + assert_eq!(mirrored_audits(store).await, before); + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("retry") + .json::() + .await + .expect("response"), + json!({"applied":true}) + ); +} + +async fn mirrored_audits(store: &ControlStore) -> Vec { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT event_id,target,record_json FROM sqb_warehouse_audit_results ORDER BY event_id", + vec![], + ) + .await + .expect("mirror"); + commit(transaction).await.expect("close read"); + rows.into_iter().map(|row| json!(row)).collect() +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 77a02110b4..1838bdf1d9 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -124,6 +124,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_audit_results", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_audit_results'", "DROP INDEX sqb_security_audit_storage_id", "ALTER TABLE sqb_security_audit DROP COLUMN storage_id", "DELETE FROM sqb_schema_migrations WHERE name='audit_storage_order'", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 68d011be0e..94065cf57c 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_audit_results", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_audit_results'", "DROP INDEX sqb_security_audit_storage_id", "ALTER TABLE sqb_security_audit DROP COLUMN storage_id", "DELETE FROM sqb_schema_migrations WHERE name='audit_storage_order'", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index bc75c07fdd..c1dd2726d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -6,7 +6,9 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; -use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryRead; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseHistoryRead, +}; use crate::bindings::_helpers::boundary::python_calls::within; use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; @@ -46,8 +48,9 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul } }; serde_json::to_string(&record) - } else if request.get("source_freshness").is_some() - || request.get("fingerprints").is_some() + } else if WarehouseHistoryKind::ALL + .iter() + .any(|kind| request.get(kind.as_str()).is_some()) { let metadata: WarehouseHistoryRead = serde_json::from_value(request).map_err(|_| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 3fcd248d91..1c7cb5cb17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -12,7 +12,7 @@ use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NA use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, + WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, }; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, @@ -23,6 +23,7 @@ use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; @@ -40,6 +41,7 @@ pub(crate) fn read_history( let location = match request { WarehouseHistoryRead::SourceFreshness(read) => &read.location, WarehouseHistoryRead::Fingerprints(read) => &read.location, + WarehouseHistoryRead::AuditResults(read) => &read.location, }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -99,6 +101,14 @@ fn read_page( request: &WarehouseHistoryRead, ) -> PyResult { let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let audit_columns: Vec = if matches!(request, WarehouseHistoryRead::AuditResults(_)) { + py.import("sqlbuild.executor.audit_results.constants")? + .getattr("AUDIT_RESULT_COLUMNS")? + .extract()? + } else { + Vec::new() + }; + let audit_columns: Vec<&str> = audit_columns.iter().map(String::as_str).collect(); let (location, after, since, limit, table, columns, timestamp) = match request { WarehouseHistoryRead::SourceFreshness(read) => ( &read.location, @@ -118,6 +128,15 @@ fn read_page( FINGERPRINT_COLUMNS.as_slice(), COLUMN_TIMESTAMP, ), + WarehouseHistoryRead::AuditResults(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_audit_results", + audit_columns.as_slice(), + "occurred_at", + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -156,13 +175,14 @@ fn read_page( ("name", name.clone()), ], )?; - let mut has_event_id = false; + let mut actual_columns: std::collections::BTreeSet = std::collections::BTreeSet::new(); for column in catalog_columns.try_iter()? { - has_event_id |= field(&column?, "name")? - .extract::()? - .eq_ignore_ascii_case("event_id"); + actual_columns.insert(field(&column?, "name")?.extract::()?.to_lowercase()); } - let mut literals = vec![None; columns.len() + 1]; + let has_event_id = actual_columns.contains("event_id"); + let audit = matches!(request, WarehouseHistoryRead::AuditResults(_)); + let key_columns: Option<&[&str]> = audit.then_some(&["result_id", "occurred_at"]); + let mut literals = vec![None; key_columns.unwrap_or(columns).len() + 1]; if let Some(after) = after { let classes = SqlValueClasses::load(py)?; for (output, value) in literals.iter_mut().zip(after) { @@ -205,11 +225,26 @@ fn read_page( .extract::() }) .transpose()?; + let projection: Vec = columns + .iter() + .map(|column| { + if audit + && matches!(*column, "audit_description" | "audit_meta_json") + && !actual_columns.contains(*column) + { + format!("NULL AS {column}") + } else { + (*column).to_owned() + } + }) + .collect(); + let projection: Vec<&str> = projection.iter().map(String::as_str).collect(); let sql = read_history_sql( &qualified, after.map(|_| literals.as_slice()), HistoryRead { - columns, + columns: &projection, + key_columns: key_columns.or(Some(columns)), timestamp_column: timestamp, observed_since: observed_since.as_deref(), adapter: &adapter_name, @@ -236,9 +271,14 @@ fn read_page( read_count += 1; let row = row?; let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; - let mut cursor = vec![None; columns.len() + 1]; + let cursor_indexes: Vec = if audit { + vec![0, 2, columns.len()] + } else { + (0..columns.len() + 1).collect() + }; + let mut cursor = vec![None; cursor_indexes.len()]; for (index, value) in cursor.iter_mut().enumerate() { - let raw = row.get_item(index)?; + let raw = row.get_item(cursor_indexes[index])?; *value = if raw.is_none() { None } else { @@ -248,6 +288,7 @@ fn read_page( let observed = match &record { WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, WarehouseHistoryRecord::Fingerprint(record) => record.ts, + WarehouseHistoryRecord::AuditResult(record) => record.occurred_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -281,6 +322,9 @@ fn decode_record( qualified, )?, ), + WarehouseHistoryRead::AuditResults(read) => { + return decode_audit_record(&read.location, row, (columns, timestamp)); + } }; let mut value = json!({ "state_database": location.database, @@ -306,13 +350,41 @@ fn decode_record( json!(item.extract::()?) }; } - match request { - WarehouseHistoryRead::SourceFreshness(_) => { - serde_json::from_value(value).map(WarehouseHistoryRecord::SourceFreshness) - } - WarehouseHistoryRead::Fingerprints(_) => { - serde_json::from_value(value).map(WarehouseHistoryRecord::Fingerprint) - } + if matches!(request, WarehouseHistoryRead::SourceFreshness(_)) { + serde_json::from_value(value).map(WarehouseHistoryRecord::SourceFreshness) + } else { + serde_json::from_value(value).map(WarehouseHistoryRecord::Fingerprint) } .map_err(|error| PyValueError::new_err(error.to_string())) } + +fn decode_audit_record( + location: &WarehouseStateLocation, + row: &Bound<'_, PyAny>, + (columns, timestamp): (&[&str], &str), +) -> PyResult { + let mut value = json!({"state_database": location.database, "state_schema": location.schema}); + for (index, name) in columns.iter().enumerate() { + let item = row.get_item(index)?; + value[*name] = if *name == timestamp { + let item = if item.is_instance_of::() { + item.py() + .import("datetime")? + .getattr("datetime")? + .call_method1("fromisoformat", (&item,))? + } else { + item + }; + json!( + presumed_utc_datetime(&item)? + .call_method0("isoformat")? + .extract::()? + ) + } else { + plain_value(&item)? + }; + } + serde_json::from_value(value) + .map(WarehouseHistoryRecord::AuditResult) + .map_err(|error| PyValueError::new_err(error.to_string())) +} diff --git a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs index 2d871377ba..8a415a7352 100644 --- a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -11,19 +11,20 @@ pub fn read_history_sql( ) -> Result { let HistoryRead { columns, + key_columns, timestamp_column, observed_since, adapter, has_event_id, limit, } = read; - let mut columns = columns.to_vec(); let mut projection = columns.join(", "); projection.push_str(if has_event_id { ", event_id" } else { ", NULL AS event_id" }); + let mut columns = key_columns.unwrap_or(columns).to_vec(); columns.push(if has_event_id { "event_id" } else { "NULL" }); let ordered = columns .iter() diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs index 8c8a1f5792..5c41f277a2 100644 --- a/crates/sqlbuild-state/src/state_tables/models.rs +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -12,6 +12,8 @@ pub struct TableColumn { #[derive(Clone, Copy, Debug)] pub struct HistoryRead<'a> { pub columns: &'a [&'a str], + /// Stable identity and timestamp keys omit numeric or evidence payloads from ordering. + pub key_columns: Option<&'a [&'a str]>, pub timestamp_column: &'a str, pub observed_since: Option<&'a str>, pub adapter: &'a str, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a516ba325a..3336692348 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -13,6 +13,7 @@ import time from collections.abc import Callable, Iterator from contextlib import AbstractContextManager, closing, contextmanager, nullcontext, suppress +from dataclasses import replace from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path @@ -27,6 +28,8 @@ from sqlbuild._native import Fingerprint, write_fingerprint from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter +from sqlbuild.executor.audit_results.main._write import write_audit_result_records +from sqlbuild.executor.audit_results.models import AuditResultRecord from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -38,6 +41,7 @@ docker_deployment, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +from tests.integration.src.sqlbuild.executor.audit_results.main.helpers import build_record class RejectedPublicationOutput(StringIO): @@ -107,6 +111,34 @@ def append_legacy_history( _ = adapter.execute(connection=connection, sql=statement.format(**row)) +def append_audit_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Use the public warehouse writer, including duplicate IDs and an external CLI run.""" + record: AuditResultRecord = replace( + build_record(), + run_id="", + evidence_json=fixture["record"]["evidence_json"], + evidence_count=1, + evidence_truncated=True, + audit_meta_json=fixture["record"]["audit_meta_json"], + ) + records: list[AuditResultRecord] = [ + replace(record, result_id=f"result-{index:03}") for index in range(fixture["source_rows"]) + ] + records.extend([records[0]] * fixture["duplicate_rows"]) + records.append(replace(records[0], occurred_at=datetime.fromisoformat(fixture["later_time"]))) + write_audit_result_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=records, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_audit_result_table_sql, + render_create_index_sqls=adapter.render_create_audit_result_index_sqls, + ) + + def warehouse_history_page( project: Path, table: str, @@ -147,6 +179,23 @@ def warehouse_history_page( return json.loads(result.stdout) +def warehouse_history_pages( + project: Path, table: str, observed_since: str | None = None +) -> list[dict[str, Any]]: + """Follow bounded source pages, including empty filtered pages with a continuation.""" + pages: list[dict[str, Any]] = [] + after: list[str | None] | None = None + more: bool = True + while more: + page: dict[str, Any] = warehouse_history_page(project, table, after, observed_since) + assert page is not None + pages.append(page) + after = page["next_after"] + more = after is not None + assert len(pages) <= 4, "history cursor must advance past duplicate physical rows" + return pages + + def sync_warehouse_history(api: SnapshotApi, table: str, batch: dict[str, Any]) -> dict[str, Any]: """Use the public location transport and return its bounded reconciliation continuation.""" request: Request = Request( @@ -218,7 +267,12 @@ def wait_for_warehouse_sync_health( datetime.fromisoformat(report["received_at"]) > after, ) by_outcome.setdefault(key, []).append(report) - reports = by_outcome.get(("warehouse_sync", expected["failure"], True), []) + reports = list( + filter( + lambda report: report["report"]["health"]["table"] in expected["tables"], + by_outcome.get(("warehouse_sync", expected["failure"], True), []), + ) + ) assert time.monotonic() < deadline, (reports, log.read_text()) time.sleep(0.05) for report in reports: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index b0a734a935..3071a8dec3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -42,6 +42,7 @@ WarehouseHistoryTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + append_audit_history, append_contended_fingerprints, append_legacy_history, append_retried_fingerprints, @@ -52,10 +53,162 @@ wait_for_generation, wait_for_warehouse_sync_health, warehouse_history_page, + warehouse_history_pages, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +@pytest.mark.parametrize( + "test_case", + [ + CatalogConnectionTestCase("DuckDB audit history", "duckdb", "database", ""), + CatalogConnectionTestCase("SQLite audit history", "sqlite", "path", ""), + ], + ids=lambda case: case.description, +) +def test_given_external_audit_results_when_reading_legacy_pages_then_duplicates_text_and_timestamps_survive( + test_case: CatalogConnectionTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json").read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + append_audit_history(adapter, connection, fixture) + for column in ("audit_description", "audit_meta_json"): + _ = adapter.execute( + connection=connection, sql=f"ALTER TABLE _sqlbuild_audit_results DROP COLUMN {column}" + ) + adapter.close(connection) + pages: list[dict[str, Any]] = warehouse_history_pages(project, "audit_results") + rows: list[dict[str, Any]] = [] + for page in pages: + rows.extend(page["records"]) + assert [len(page["records"]) for page in pages] == [100, 100, 4] + assert {row["result_id"] for row in rows} == { + f"result-{index:03}" for index in range(fixture["source_rows"]) + } + expected: dict[str, Any] = fixture["record"] | { + "audit_description": None, + "audit_meta_json": None, + } + assert rows[0] == expected + assert rows[0]["run_id"] == test_case.expected_run_id + assert rows[100] == expected | {"occurred_at": fixture["later_time"]} + assert rows[-1] == expected | {"result_id": fixture["retained_result"]} + recent_pages: list[dict[str, Any]] = warehouse_history_pages( + project, "audit_results", fixture["later_time"] + ) + recent_rows: list[dict[str, Any]] = [] + for page in recent_pages: + recent_rows.extend(page["records"]) + assert recent_rows == [expected | {"occurred_at": fixture["later_time"]}] + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + columns: set[str] = { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_audit_results" + ) + } + assert not columns & {"audit_description", "audit_meta_json"}, ( + "the reader never upgrades source storage" + ) + assert adapter.query( + connection=connection, sql="SELECT COUNT(*) FROM _sqlbuild_audit_results", limit=1 + ).rows == ((fixture["source_rows"] + fixture["duplicate_rows"] + 1,),) + adapter.close(connection) + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogConnectionTestCase("DuckDB automatic audit mirror", "duckdb", "database", ""), + CatalogConnectionTestCase("SQLite automatic audit mirror", "sqlite", "path", ""), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_audit_history_is_rebuilt_and_retained_then_only_complete_scans_delete( + test_case: CatalogConnectionTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json").read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + append_audit_history(adapter, connection, fixture) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) + with snapshot_api(tmp_path / "control") as api: + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ + | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "30"}, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + identities: dict[str, str] = {} + all_ids: set[str] = {f"result-{index:03}" for index in range(fixture["source_rows"])} + for expected_ids, mirror_sql, warehouse_sql in ( + (all_ids, "DELETE FROM sqb_warehouse_audit_results", "SELECT 1"), + ( + all_ids, + "SELECT 1", + "DELETE FROM _sqlbuild_audit_results " + f"WHERE result_id <> '{fixture['retained_result']}'", + ), + ({fixture["retained_result"]}, "SELECT 1", "SELECT 1"), + ): + deadline: float = time.monotonic() + 45 + mirrored: list[tuple[str, str]] = [] + while len(mirrored) != len(expected_ids): + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with closing(sqlite3.connect(control)) as database: + mirrored = database.execute( + "SELECT event_id,record_json FROM sqb_warehouse_audit_results ORDER BY event_id" + ).fetchall() + time.sleep(0.05) + assert {json.loads(row)["result_id"] for _, row in mirrored} == expected_ids + for event_id, text in mirrored: + record: dict[str, Any] = json.loads(text) + assert record == fixture["record"] | {"result_id": record["result_id"]} + assert record["run_id"] == test_case.expected_run_id + assert identities.setdefault(record["result_id"], event_id) == event_id + with closing(sqlite3.connect(control)) as database: + _ = database.execute(mirror_sql) + database.commit() + connection = adapter.connect( + {test_case.connection_key: str(project / "warehouse.db")} + ) + _ = adapter.execute(connection=connection, sql=warehouse_sql) + adapter.close(connection) + + @pytest.mark.parametrize( "test_case", [ From be4cefa996f55aaee4640fd2797306ab57d03fad Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:21:56 +0000 Subject: [PATCH 091/138] feat: synchronize warehouse janitor history --- .../orchestration/audit_result_history.json | 9 + .../orchestration/janitor_event_history.json | 33 ++++ clients/typescript/src/schema.d.ts | 95 ++++++++++- clients/typescript/tests/types.ts | 8 + .../src/native_commands/_helpers/execution.rs | 9 + .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 22 ++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 12 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 4 +- .../src/store/warehouse_janitor_events.sql | 16 ++ .../src/warehouse_metadata/_helpers/events.rs | 111 ++++++++++++ .../src/warehouse_metadata/_helpers/mod.rs | 1 + .../main/ingest_audit_results.rs | 105 +++++------- .../main/ingest_janitor_events.rs | 76 +++++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 66 +++++++- .../tests/api/test_contracts.rs | 4 +- .../tests/api/test_warehouse_metadata.rs | 29 +++- .../{audit_results => events}/helpers.rs | 140 +++++++-------- .../tests/credentials/test_audit.rs | 2 + .../tests/store/helpers.rs | 2 + .../native_host/warehouse_metadata.rs | 72 +++++--- fensu.toml | 4 + .../publication/_test_types.py | 9 + .../compiled_project/publication/helpers.py | 41 +++++ .../publication/test_warehouse_metadata.py | 159 +++++++++++++----- 28 files changed, 822 insertions(+), 214 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_janitor_events.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs rename crates/sqlbuild-orchestration/tests/api/warehouse_metadata/{audit_results => events}/helpers.rs (61%) diff --git a/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json b/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json index acab1bf063..5fef8086b6 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json +++ b/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json @@ -47,5 +47,14 @@ "duplicate_rows": 105, "later_time": "2026-02-03T04:05:06.000001Z", "retained_result": "result-103", + "kind": "audit_results", + "source_key": "result_id", + "id_prefix": "result", + "page_lengths": [100, 100, 4], + "later_index": 100, + "later_record": {"occurred_at": "2026-02-03T04:05:06.000001Z"}, + "last_record": {"result_id": "result-103"}, + "record_overrides": {}, + "source_count": 210, "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json b/benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json new file mode 100644 index 0000000000..b3fed6df80 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json @@ -0,0 +1,33 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "event-000", + "schema_version": 1, + "event_type": "archive", + "occurred_at": "2026-02-03T04:05:06Z", + "run_id": "cleanup-run", + "relation_database": null, + "relation_schema": "main", + "relation_type": "BASE TABLE", + "original_name": "orders", + "original_qualified_name": "main.orders", + "archive_name": "_sqb_archive_orders_🧾", + "archive_qualified_name": "main.\"_sqb_archive_orders_🧾\"", + "archived_at": "2026-01-02T03:04:05Z" + }, + "source_rows": 104, + "duplicate_rows": 105, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "event-103", + "kind": "janitor_events", + "source_key": "event_id", + "id_prefix": "event", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": {"event_id": "event-103", "event_type": "delete", "occurred_at": "2026-02-03T04:05:06.000001Z"}, + "last_record": {"event_id": "event-103", "event_type": "delete", "occurred_at": "2026-02-03T04:05:06.000001Z"}, + "record_overrides": {"event-103": {"event_type": "delete", "occurred_at": "2026-02-03T04:05:06.000001Z"}}, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 9bee33e5bf..390b83ee51 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -307,6 +307,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/janitor-events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_janitor_events"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/source-freshness": { parameters: { query?: never; @@ -709,6 +725,14 @@ export interface components { failed: components["schemas"]["Table"][]; failure?: string | null; }; + JanitorEventsBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseJanitorRecord"][]; + /** Format: uuid */ + session_id: string; + }; /** @description A trusted operator names a published job; execution inputs are never supplied by the caller. */ JobRunRequest: { job_key: string; @@ -1431,7 +1455,28 @@ export interface components { * @description Only immutable history tables support a timestamp watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events"; + /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ + WarehouseJanitorRecord: { + archive_name: string; + archive_qualified_name: string; + /** Format: date-time */ + archived_at: string; + event_id: string; + event_type: string; + /** Format: date-time */ + occurred_at: string; + original_name?: string | null; + original_qualified_name?: string | null; + relation_database?: string | null; + relation_schema: string; + relation_type?: string | null; + run_id: string; + /** Format: int64 */ + schema_version: number; + state_database?: string | null; + state_schema: string; + }; WarehouseReconciliationCursor: { /** Format: uuid */ event_id: string; @@ -2248,6 +2293,54 @@ export interface operations { }; }; }; + report_janitor_events: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["JanitorEventsBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_source_freshness: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 10121c006c..612bfb4953 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -31,6 +31,14 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/audit-r params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const janitorHistory: components["schemas"]["WarehouseHistoryKind"] = "janitor_events"; +const archivedAt: components["schemas"]["WarehouseJanitorRecord"]["archived_at"] = "2026-01-02T03:04:05Z"; +void janitorHistory; +void archivedAt; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/janitor-events", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index cd862f286d..0093697096 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -424,6 +424,15 @@ pub(crate) async fn deliver_metadata_page( ) .await } + WarehouseHistoryKind::JanitorEvents => { + client + .sync_janitor_events( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid janitor events batch"))?, + ) + .await + } } } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index c4090b0fd6..f59703b526 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -43,6 +43,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_janitor_events(&store, (location_id, &target), &batch).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 8a007c72d7..039f320df3 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -51,6 +51,7 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report, warehouse_metadata::report_fingerprints, warehouse_metadata::report_audit_results, + warehouse_metadata::report_janitor_events, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 0f0cd7acbf..f2d57ba30e 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,8 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - AuditResultsBatch, FingerprintBatch, SourceFreshnessBatch, WarehouseSyncResponse, - WarehouseWatermarkRead, + AuditResultsBatch, FingerprintBatch, JanitorEventsBatch, SourceFreshnessBatch, + WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -51,6 +51,14 @@ impl LocationWorkClient { self.sync_metadata((target, "audit-results"), batch).await } + pub async fn sync_janitor_events( + &self, + target: &str, + batch: &JanitorEventsBatch, + ) -> Result { + self.sync_metadata((target, "janitor-events"), batch).await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 49bbeba5ca..c7ca01f70a 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -28,6 +28,7 @@ pub(crate) const EXTERNAL_SESSIONS: &str = include_str!("../external_sessions.sq pub(crate) const WAREHOUSE_RECONCILIATION: &str = include_str!("../warehouse_reconciliation.sql"); pub(crate) const AUDIT_STORAGE_ORDER: &str = include_str!("../audit_storage_order.sql"); pub(crate) const WAREHOUSE_AUDIT_RESULTS: &str = include_str!("../warehouse_audit_results.sql"); +pub(crate) const WAREHOUSE_JANITOR_EVENTS: &str = include_str!("../warehouse_janitor_events.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 282fe89ca7..1e9b1ec8a0 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -8,7 +8,8 @@ use crate::store::_helpers::migration::{ DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, WAREHOUSE_RECONCILIATION, render, verify_migrations, + WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_RECONCILIATION, render, + verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -52,6 +53,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_reconciliation", WAREHOUSE_RECONCILIATION), ("audit_storage_order", AUDIT_STORAGE_ORDER), ("warehouse_audit_results", WAREHOUSE_AUDIT_RESULTS), + ("warehouse_janitor_events", WAREHOUSE_JANITOR_EVENTS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_janitor_events.sql b/crates/sqlbuild-orchestration/src/store/warehouse_janitor_events.sql new file mode 100644 index 0000000000..840f21e3c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_janitor_events.sql @@ -0,0 +1,16 @@ +CREATE TABLE sqb_warehouse_janitor_events ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + event_type TEXT NOT NULL, + relation_database TEXT, + relation_schema TEXT NOT NULL, + archive_name TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_janitor_events_reconciliation ON sqb_warehouse_janitor_events (location_id,target,state_database,state_schema,observed_at,event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs new file mode 100644 index 0000000000..546f01308a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -0,0 +1,111 @@ +//! One fenced, atomic persistence path for immutable warehouse events with typed filter columns. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, scoped_event_id, validate_fields, +}; +use crate::warehouse_metadata::_helpers::reconciliation::begin_reconciliation; +use crate::warehouse_metadata::models::{StateEventsImport, WarehouseSyncResponse}; + +pub(crate) async fn ingest_events( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: StateEventsImport<'_>, +) -> Result { + let label = batch.kind.as_str().replace('_', " "); + validate_fields( + (&label, batch.generation, batch.records.len()), + batch.invalid_identity, + batch.reconciliation.is_some_and(|page| { + batch.records.iter().any(|record| { + record.location.database != page.state_database + || record.location.schema != page.state_schema + || page + .day + .is_some_and(|day| record.observed_at.date_naive() != day) + }) + }), + )?; + let table = batch.kind.mirror_table(); + let source_table = format!("_sqlbuild_{}", batch.kind.as_str()); + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await?; + if let Some(page) = batch.reconciliation { + transaction = begin_reconciliation( + transaction, + (table, location_id, target), + page, + batch.payload, + ) + .await?; + } + let mut applied = false; + let mut keys: Vec = Vec::new(); + for record in batch.records { + let event_id = scoped_event_id( + (location_id, target), + ( + record.location.database.as_deref(), + &record.location.schema, + &source_table, + ), + &record.source_id, + )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } + let mut columns = vec![ + "event_id", + "location_id", + "target", + "source_event_id", + "observed_at", + "state_database", + "state_schema", + ]; + let mut parameters = vec![ + Parameter::Id(event_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(record.source_id), + Parameter::Time(record.observed_at), + Parameter::OptionalText(record.location.database), + Parameter::Text(record.location.schema), + ]; + for (column, value) in record.columns { + columns.push(column); + parameters.push(value); + } + columns.extend(["record_json", "received_at"]); + parameters.extend([ + Parameter::Json(record.record_json), + Parameter::Time(transaction.now), + ]); + let placeholders = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let sql = format!( + "INSERT INTO {table} ({}) VALUES ({placeholders}) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + columns.join(",") + ); + let inserted; + (transaction, inserted) = query(transaction, &sql, parameters).await?; + applied |= !inserted.is_empty(); + } + commit_ingestion( + transaction, + (table, location_id, target), + (batch.reconciliation, &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs index 4c4e4c12b9..9849049bc6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod events; pub(crate) mod fingerprints; pub(crate) mod freshness; pub(crate) mod ingestion; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs index e0142e7b73..0fbe3ead87 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs @@ -4,83 +4,56 @@ use serde_json::json; use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter}; -use crate::warehouse_metadata::_helpers::ingestion::{ - begin_ingestion, commit_ingestion, scoped_event_id, validate_fields, +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + AuditResultsBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, }; -use crate::warehouse_metadata::_helpers::reconciliation::begin_reconciliation; -use crate::warehouse_metadata::models::{AuditResultsBatch, WarehouseSyncResponse}; pub async fn ingest_audit_results( store: &ControlStore, - (location_id, target): (Uuid, &str), + scope: (Uuid, &str), batch: &AuditResultsBatch, ) -> Result { validate_fields( ("audit results", batch.generation, batch.records.len()), - batch - .records - .iter() - .flat_map(|record| [&record.result_id, &record.state_schema, &record.audit_name]) - .any(|value| value.trim().is_empty()), - batch.reconciliation.as_ref().is_some_and(|page| { - batch.records.iter().any(|record| { - record.state_database != page.state_database - || record.state_schema != page.state_schema - || page - .day - .is_some_and(|day| record.occurred_at.date_naive() != day) - }) - }), + false, + false, )?; - let mut transaction = begin_ingestion( + ingest_events( store, - (location_id, target), - (batch.session_id, batch.generation), - ) - .await?; - if let Some(page) = &batch.reconciliation { - transaction = begin_reconciliation( - transaction, - ("sqb_warehouse_audit_results", location_id, target), - page, - json!(batch), - ) - .await?; - } - let mut applied = false; - let mut keys: Vec = Vec::new(); - for record in &batch.records { - let event_id = scoped_event_id( - (location_id, target), - ( - record.state_database.as_deref(), - &record.state_schema, - "_sqlbuild_audit_results", - ), - &record.result_id, - )?; - if batch.reconciliation.is_some() { - keys.push(event_id); - } - let now = transaction.now; - let inserted; - (transaction, inserted) = query(transaction, - "INSERT INTO sqb_warehouse_audit_results (event_id,location_id,target,source_event_id,observed_at,state_database,state_schema,audit_name,outcome,record_json,received_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", - vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), - Parameter::Text(record.result_id.clone()), Parameter::Time(record.occurred_at), - Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), - Parameter::Text(record.audit_name.clone()), Parameter::Text(record.outcome.clone()), - Parameter::Json(json!(record)), Parameter::Time(now)]).await?; - applied |= !inserted.is_empty(); - } - commit_ingestion( - transaction, - ("sqb_warehouse_audit_results", location_id, target), - (batch.reconciliation.as_ref(), &keys), - applied, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::AuditResults, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| [&record.result_id, &record.state_schema, &record.audit_name]) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.result_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.occurred_at, + columns: vec![ + ("audit_name", Parameter::Text(record.audit_name.clone())), + ("outcome", Parameter::Text(record.outcome.clone())), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, ) .await } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs new file mode 100644 index 0000000000..b00ac733d7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs @@ -0,0 +1,76 @@ +//! Mirror archive/delete history; the warehouse remains the owner of physical cleanup. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + JanitorEventsBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, +}; + +pub async fn ingest_janitor_events( + store: &ControlStore, + scope: (Uuid, &str), + batch: &JanitorEventsBatch, +) -> Result { + validate_fields( + ("janitor events", batch.generation, batch.records.len()), + false, + false, + )?; + ingest_events( + store, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::JanitorEvents, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.relation_schema, + &record.event_type, + &record.archive_name, + &record.archive_qualified_name, + ] + }) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.occurred_at, + columns: vec![ + ("event_type", Parameter::Text(record.event_type.clone())), + ( + "relation_database", + Parameter::OptionalText(record.relation_database.clone()), + ), + ( + "relation_schema", + Parameter::Text(record.relation_schema.clone()), + ), + ("archive_name", Parameter::Text(record.archive_name.clone())), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 31a249d862..a9cd5bb8a7 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,4 +1,5 @@ pub mod ingest_audit_results; pub mod ingest_fingerprints; +pub mod ingest_janitor_events; pub mod ingest_source_freshness; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 610d9cfc47..4ef54db78c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -92,6 +92,37 @@ pub struct AuditResultsBatch { pub reconciliation: Option, } +/// Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseJanitorRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub schema_version: i64, + pub event_type: String, + pub occurred_at: DateTime, + pub run_id: String, + pub relation_database: Option, + pub relation_schema: String, + pub relation_type: Option, + pub original_name: Option, + pub original_qualified_name: Option, + pub archive_name: String, + pub archive_qualified_name: String, + pub archived_at: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct JanitorEventsBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + /// Physical namespace of SQLBuild's source-freshness state table, never a source query. #[derive( Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, @@ -109,13 +140,15 @@ pub enum WarehouseHistoryKind { Fingerprints, SourceFreshness, AuditResults, + JanitorEvents, } impl WarehouseHistoryKind { - pub const ALL: [Self; 3] = [ + pub const ALL: [Self; 4] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, + Self::JanitorEvents, ]; #[must_use] @@ -124,6 +157,7 @@ impl WarehouseHistoryKind { Self::Fingerprints => "fingerprints", Self::SourceFreshness => "source_freshness", Self::AuditResults => "audit_results", + Self::JanitorEvents => "janitor_events", } } @@ -133,6 +167,7 @@ impl WarehouseHistoryKind { Self::Fingerprints => "sqb_warehouse_fingerprints", Self::SourceFreshness => "sqb_warehouse_freshness", Self::AuditResults => "sqb_warehouse_audit_results", + Self::JanitorEvents => "sqb_warehouse_janitor_events", } } } @@ -227,10 +262,10 @@ pub struct FingerprintRead { pub limit: Option, } -/// Duplicate result IDs may be appended; timestamps distinguish their ordered observations. +/// Stable event identities and timestamps order observations without comparing their payloads. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct AuditResultsRead { +pub struct StateEventsRead { pub location: WarehouseStateLocation, pub after: Option<[Option; 3]>, pub observed_since: Option>, @@ -243,7 +278,8 @@ pub struct AuditResultsRead { pub enum WarehouseHistoryRead { SourceFreshness(SourceFreshnessRead), Fingerprints(FingerprintRead), - AuditResults(AuditResultsRead), + AuditResults(StateEventsRead), + JanitorEvents(StateEventsRead), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -252,6 +288,7 @@ pub enum WarehouseHistoryRecord { SourceFreshness(WarehouseFreshnessRecord), Fingerprint(WarehouseFingerprintRecord), AuditResult(Box), + JanitorEvent(WarehouseJanitorRecord), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -260,3 +297,24 @@ pub struct WarehouseHistoryPage { pub records: Vec, pub next_after: Option>>, } + +/// Typed import columns are selected by code, never from location-supplied field names. +#[derive(Debug)] +pub(crate) struct StateEventImport { + pub source_id: String, + pub location: WarehouseStateLocation, + pub observed_at: DateTime, + pub columns: Vec<(&'static str, crate::store::models::Parameter)>, + pub record_json: serde_json::Value, +} + +#[derive(Debug)] +pub(crate) struct StateEventsImport<'a> { + pub kind: WarehouseHistoryKind, + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + pub invalid_identity: bool, + pub reconciliation: Option<&'a WarehouseReconciliationPage>, + pub payload: serde_json::Value, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index c303668478..5a14957b9e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -38,8 +38,8 @@ mod test_types; mod test_warehouse_metadata; #[path = "test_worker.rs"] mod test_worker; -#[path = "warehouse_metadata/audit_results/helpers.rs"] -mod warehouse_audit_results_helpers; +#[path = "warehouse_metadata/events/helpers.rs"] +mod warehouse_events_helpers; #[path = "warehouse_metadata/helpers.rs"] mod warehouse_metadata_helpers; #[path = "warehouse_metadata/reconciliation/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 7f2accc76c..e77332366b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -24,7 +24,7 @@ use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::models::SourceFreshnessBatch; +use crate::warehouse_metadata::models::{SourceFreshnessBatch, WarehouseHistoryKind}; #[test] fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicity_are_preserved() { @@ -322,13 +322,26 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit (session_id, preview_session), ) .await; - crate::api::tests::warehouse_audit_results_helpers::audit_history( - store, - &outbound, - (session_id, preview_session), - (&client, &url, &token.bearer), - ) - .await; + for event_case in [ + ( + WarehouseHistoryKind::AuditResults, + "audit_result_history", + "result_id", + ), + ( + WarehouseHistoryKind::JanitorEvents, + "janitor_event_history", + "event_id", + ), + ] { + crate::api::tests::warehouse_events_helpers::event_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer, event_case), + ) + .await; + } stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs similarity index 61% rename from crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs rename to crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs index a39992e181..d5defa43b2 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/audit_results/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -1,4 +1,4 @@ -//! External audit facts retain their identities, physical scopes and atomic replay semantics. +//! Warehouse events retain their identities, physical scopes and atomic replay semantics. use reqwest::{Client, StatusCode}; use serde_json::{Value, json}; @@ -11,42 +11,45 @@ use crate::store::main::{ use crate::store::models::{ControlStore, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; -use crate::warehouse_metadata::models::{AuditResultsBatch, WarehouseWatermarkRead}; +use crate::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseSyncResponse, WarehouseWatermarkRead, +}; -pub(crate) async fn audit_history( +pub(crate) async fn event_history( store: &ControlStore, outbound: &LocationWorkClient, (session_id, preview_session): (Uuid, Uuid), - (client, freshness_url, bearer): (&Client, &str, &str), + (client, freshness_url, bearer, (kind, fixture_name, source_key)): ( + &Client, + &str, + &str, + (WarehouseHistoryKind, &str, &str), + ), ) { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "../../benchmarks/duckshop/edge_cases/orchestration/{fixture_name}.json" + )); let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) - .expect("audit history"); + .expect("event history"); let record = fixture["record"].clone(); let batch = json!({"session_id": session_id, "generation": 1, "records": [record, record]}); - let typed: AuditResultsBatch = serde_json::from_value(batch.clone()).expect("audit batch"); + let url = freshness_url.replace("source-freshness", &kind.as_str().replace('_', "-")); for expected in [true, false] { assert_eq!( - outbound - .sync_audit_results("development", &typed) - .await - .expect("sync") - .applied, + sync_events((client, &url, bearer), &batch).await.applied, expected ); } - let before = mirrored_audits(store).await; + let before = mirrored_events(store, kind).await; assert_eq!( before.len(), 1, - "duplicate deterministic result IDs deduplicate" + "duplicate deterministic event identities deduplicate" ); assert_eq!( before[0]["record_json"], record, - "empty external run ID and warehouse text survive" + "external run IDs, text and timestamps survive" ); - let url = freshness_url.replace("source-freshness", "audit-results"); for (field, value, status) in [ ("generation", json!(2), StatusCode::CONFLICT), ( @@ -102,29 +105,22 @@ pub(crate) async fn audit_history( scoped["state_schema"] = json!("reporting"); let mut database = record.clone(); database["state_database"] = json!("analytics"); - let scopes: AuditResultsBatch = serde_json::from_value(json!({"session_id":session_id, - "generation":1, "records":[scoped,database]})) - .expect("physical scopes"); + let scopes = json!({"session_id":session_id, + "generation":1, "records":[scoped,database]}); + assert!(sync_events((client, &url, bearer), &scopes).await.applied); + let preview = json!({"session_id":preview_session, + "generation":1, "records":[record]}); assert!( - outbound - .sync_audit_results("development", &scopes) - .await - .expect("scope") - .applied + sync_events( + (client, &url.replace("/development/", "/preview/"), bearer), + &preview + ) + .await + .applied ); - let preview: AuditResultsBatch = serde_json::from_value(json!({"session_id":preview_session, - "generation":1, "records":[record]})) - .expect("preview"); - assert!( - outbound - .sync_audit_results("preview", &preview) - .await - .expect("target scope") - .applied - ); - assert_eq!(mirrored_audits(store).await.len(), 4); + assert_eq!(mirrored_events(store, kind).await.len(), 4); let watermark: WarehouseWatermarkRead = serde_json::from_value(json!({"session_id":session_id, - "generation":1,"kind":"audit_results","location":{"database":null,"schema":"main"}})) + "generation":1,"kind":kind,"location":{"database":null,"schema":"main"}})) .expect("watermark"); assert_eq!( json!( @@ -136,18 +132,18 @@ pub(crate) async fn audit_history( record["occurred_at"] ); let fault = TransactionFault { - description: "audit page rolls back atomically", - table: "sqb_warehouse_audit_results", + description: "event page rolls back atomically", + table: kind.mirror_table(), operation: "INSERT", condition: "WHEN NEW.source_event_id='rejected-result'", sqlite_body: "SELECT RAISE(ABORT, 'injected audit failure')", postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; - let before = mirrored_audits(store).await; + let before = mirrored_events(store, kind).await; let mut first = record.clone(); - first["result_id"] = json!("accepted-result"); + first[source_key] = json!("accepted-result"); let mut second = record.clone(); - second["result_id"] = json!("rejected-result"); + second[source_key] = json!("rejected-result"); let additional = json!({"session_id":session_id,"generation":1,"records":[first,second]}); install_fault(store, &fault).await; assert_eq!( @@ -163,46 +159,35 @@ pub(crate) async fn audit_history( ); remove_fault(store, &fault).await; assert_eq!( - mirrored_audits(store).await, + mirrored_events(store, kind).await, before, "no partial page survives" ); let (transaction, _) = query( begin(store, TransactionMode::Write).await.expect("write"), - "DELETE FROM sqb_warehouse_audit_results", + &format!("DELETE FROM {}", kind.mirror_table()), vec![], ) .await .expect("empty mirror"); commit(transaction).await.expect("wipe"); + assert!(sync_events((client, &url, bearer), &batch).await.applied); + assert!(sync_events((client, &url, bearer), &scopes).await.applied); assert!( - outbound - .sync_audit_results("development", &typed) - .await - .expect("rebuild") - .applied - ); - assert!( - outbound - .sync_audit_results("development", &scopes) - .await - .expect("rebuild scopes") - .applied - ); - assert!( - outbound - .sync_audit_results("preview", &preview) - .await - .expect("rebuild target") - .applied + sync_events( + (client, &url.replace("/development/", "/preview/"), bearer), + &preview + ) + .await + .applied ); assert_eq!( - mirrored_audits(store).await, + mirrored_events(store, kind).await, before, "identical facts after rebuild" ); migrate_store(store).await.expect("warm migration"); - assert_eq!(mirrored_audits(store).await, before); + assert_eq!(mirrored_events(store, kind).await, before); assert_eq!( client .post(&url) @@ -218,10 +203,13 @@ pub(crate) async fn audit_history( ); } -async fn mirrored_audits(store: &ControlStore) -> Vec { +async fn mirrored_events(store: &ControlStore, kind: WarehouseHistoryKind) -> Vec { let (transaction, rows) = query( begin(store, TransactionMode::Read).await.expect("read"), - "SELECT event_id,target,record_json FROM sqb_warehouse_audit_results ORDER BY event_id", + &format!( + "SELECT event_id,target,record_json FROM {} ORDER BY event_id", + kind.mirror_table() + ), vec![], ) .await @@ -229,3 +217,21 @@ async fn mirrored_audits(store: &ControlStore) -> Vec { commit(transaction).await.expect("close read"); rows.into_iter().map(|row| json!(row)).collect() } + +async fn sync_events( + (client, url, bearer): (&Client, &str, &str), + batch: &Value, +) -> WarehouseSyncResponse { + client + .post(url) + .bearer_auth(bearer) + .json(batch) + .send() + .await + .expect("event sync") + .error_for_status() + .expect("accepted page") + .json() + .await + .expect("sync response") +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 1838bdf1d9..c5ae497678 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -124,6 +124,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_janitor_events", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_janitor_events'", "DROP TABLE sqb_warehouse_audit_results", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_audit_results'", "DROP INDEX sqb_security_audit_storage_id", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 94065cf57c..c3312cc506 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_janitor_events", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_janitor_events'", "DROP TABLE sqb_warehouse_audit_results", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_audit_results'", "DROP INDEX sqb_security_audit_storage_id", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 1c7cb5cb17..a772cf1e73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -41,7 +41,9 @@ pub(crate) fn read_history( let location = match request { WarehouseHistoryRead::SourceFreshness(read) => &read.location, WarehouseHistoryRead::Fingerprints(read) => &read.location, - WarehouseHistoryRead::AuditResults(read) => &read.location, + WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::JanitorEvents(read) => { + &read.location + } }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -101,14 +103,18 @@ fn read_page( request: &WarehouseHistoryRead, ) -> PyResult { let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); - let audit_columns: Vec = if matches!(request, WarehouseHistoryRead::AuditResults(_)) { - py.import("sqlbuild.executor.audit_results.constants")? + let event_columns: Vec = match request { + WarehouseHistoryRead::AuditResults(_) => py + .import("sqlbuild.executor.audit_results.constants")? .getattr("AUDIT_RESULT_COLUMNS")? - .extract()? - } else { - Vec::new() + .extract()?, + WarehouseHistoryRead::JanitorEvents(_) => py + .import("sqlbuild.executor.janitor_events.constants")? + .getattr("JANITOR_EVENT_COLUMNS")? + .extract()?, + _ => Vec::new(), }; - let audit_columns: Vec<&str> = audit_columns.iter().map(String::as_str).collect(); + let event_columns: Vec<&str> = event_columns.iter().map(String::as_str).collect(); let (location, after, since, limit, table, columns, timestamp) = match request { WarehouseHistoryRead::SourceFreshness(read) => ( &read.location, @@ -134,7 +140,16 @@ fn read_page( read.observed_since, read.limit, "_sqlbuild_audit_results", - audit_columns.as_slice(), + event_columns.as_slice(), + "occurred_at", + ), + WarehouseHistoryRead::JanitorEvents(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_janitor_events", + event_columns.as_slice(), "occurred_at", ), }; @@ -181,7 +196,21 @@ fn read_page( } let has_event_id = actual_columns.contains("event_id"); let audit = matches!(request, WarehouseHistoryRead::AuditResults(_)); - let key_columns: Option<&[&str]> = audit.then_some(&["result_id", "occurred_at"]); + let key_columns: Option<&[&str]> = match request { + WarehouseHistoryRead::AuditResults(_) => Some(&["result_id", "occurred_at"]), + WarehouseHistoryRead::JanitorEvents(_) => Some(&["event_id", "occurred_at"]), + _ => None, + }; + let mut cursor_indexes: Vec = columns + .iter() + .enumerate() + .filter_map(|(index, name)| { + key_columns + .is_none_or(|keys| keys.contains(name)) + .then_some(index) + }) + .collect(); + cursor_indexes.push(columns.len()); let mut literals = vec![None; key_columns.unwrap_or(columns).len() + 1]; if let Some(after) = after { let classes = SqlValueClasses::load(py)?; @@ -271,11 +300,6 @@ fn read_page( read_count += 1; let row = row?; let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; - let cursor_indexes: Vec = if audit { - vec![0, 2, columns.len()] - } else { - (0..columns.len() + 1).collect() - }; let mut cursor = vec![None; cursor_indexes.len()]; for (index, value) in cursor.iter_mut().enumerate() { let raw = row.get_item(cursor_indexes[index])?; @@ -289,6 +313,7 @@ fn read_page( WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, WarehouseHistoryRecord::Fingerprint(record) => record.ts, WarehouseHistoryRecord::AuditResult(record) => record.occurred_at, + WarehouseHistoryRecord::JanitorEvent(record) => record.occurred_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -323,7 +348,14 @@ fn decode_record( )?, ), WarehouseHistoryRead::AuditResults(read) => { - return decode_audit_record(&read.location, row, (columns, timestamp)); + return decode_event_record(&read.location, row, (columns, &[timestamp])); + } + WarehouseHistoryRead::JanitorEvents(read) => { + return decode_event_record( + &read.location, + row, + (columns, &[timestamp, "archived_at"]), + ); } }; let mut value = json!({ @@ -358,15 +390,15 @@ fn decode_record( .map_err(|error| PyValueError::new_err(error.to_string())) } -fn decode_audit_record( +fn decode_event_record( location: &WarehouseStateLocation, row: &Bound<'_, PyAny>, - (columns, timestamp): (&[&str], &str), + (columns, timestamps): (&[&str], &[&str]), ) -> PyResult { let mut value = json!({"state_database": location.database, "state_schema": location.schema}); for (index, name) in columns.iter().enumerate() { let item = row.get_item(index)?; - value[*name] = if *name == timestamp { + value[*name] = if timestamps.contains(name) { let item = if item.is_instance_of::() { item.py() .import("datetime")? @@ -384,7 +416,5 @@ fn decode_audit_record( plain_value(&item)? }; } - serde_json::from_value(value) - .map(WarehouseHistoryRecord::AuditResult) - .map_err(|error| PyValueError::new_err(error.to_string())) + serde_json::from_value(value).map_err(|error| PyValueError::new_err(error.to_string())) } diff --git a/fensu.toml b/fensu.toml index a1bd617846..95e23d4d31 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1661,6 +1661,10 @@ reason = "Command document projections intentionally mirror the public envelope paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs"] reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs"] +reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." + [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/migrations/_helpers/old_name_views.py" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 5305d54551..d8b325146a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -176,6 +176,15 @@ class CatalogConnectionTestCase: expected_run_id: str +@dataclass(frozen=True) +class StateEventsTestCase(CatalogConnectionTestCase): + description: str + expected_run_id: str + fixture_name: str + append_history: Callable[..., None] + legacy_columns: tuple[str, ...] = () + + @dataclass(frozen=True) class AutomaticHistoryTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 3336692348..5fc97b3d4f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -30,6 +30,12 @@ from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.executor.audit_results.main._write import write_audit_result_records from sqlbuild.executor.audit_results.models import AuditResultRecord +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -139,6 +145,41 @@ def append_audit_history(adapter: Any, connection: Any, fixture: dict[str, Any]) ) +def append_janitor_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real chunked archive/delete writes, including repeated write attempts.""" + data: dict[str, Any] = fixture["record"].copy() + del data["state_database"] + del data["state_schema"] + for key in ("occurred_at", "archived_at"): + data[key] = datetime.fromisoformat(data[key]) + data["event_type"] = JanitorEventType(data["event_type"]) + record: JanitorEventRecord = JanitorEventRecord(**data) + records: list[JanitorEventRecord] = [ + replace(record, event_id=f"event-{index:03}") for index in range(fixture["source_rows"]) + ] + records[-1] = replace( + records[-1], + event_type=JanitorEventType.DELETE, + occurred_at=datetime.fromisoformat(fixture["later_time"]), + ) + _ = adapter.execute( + connection=connection, + sql=build_janitor_events_create_table_sql( + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ), + ) + writer: JanitorEventWriter = JanitorEventWriter( + execute=adapter.execute, + render_qualified_name=adapter.render_qualified_name, + run_id=record.run_id, + ) + writer.write(connection=connection, records=tuple(records)) + writer.write(connection=connection, records=(records[0],) * fixture["duplicate_rows"]) + + def warehouse_history_page( project: Path, table: str, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 3071a8dec3..b6dfb5207c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -39,11 +39,13 @@ AutomaticHistoryTestCase, CatalogConnectionTestCase, FingerprintHistoryTestCase, + StateEventsTestCase, WarehouseHistoryTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_audit_history, append_contended_fingerprints, + append_janitor_history, append_legacy_history, append_retried_fingerprints, location_process, @@ -61,65 +63,98 @@ @pytest.mark.parametrize( "test_case", [ - CatalogConnectionTestCase("DuckDB audit history", "duckdb", "database", ""), - CatalogConnectionTestCase("SQLite audit history", "sqlite", "path", ""), + StateEventsTestCase( + f"{adapter_name} {fixture_name}", + adapter_name, + connection_key, + expected_run_id, + fixture_name, + append_history, + legacy_columns, + ) + for adapter_name, connection_key, fixture_name, append_history, expected_run_id, legacy_columns in [ + ( + "duckdb", + "database", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ( + "duckdb", + "database", + "janitor_event_history", + append_janitor_history, + "cleanup-run", + (), + ), + ( + "sqlite", + "path", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + ] ], ids=lambda case: case.description, ) -def test_given_external_audit_results_when_reading_legacy_pages_then_duplicates_text_and_timestamps_survive( - test_case: CatalogConnectionTestCase, tmp_path: Path +def test_given_external_state_events_when_reading_pages_then_identities_text_and_timestamps_survive( + test_case: StateEventsTestCase, tmp_path: Path ) -> None: project: Path = tmp_path / "project" fixture: dict[str, Any] = json.loads( - Path("benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json").read_text() + Path( + f"benchmarks/duckshop/edge_cases/orchestration/{test_case.fixture_name}.json" + ).read_text() ) adapter: Any connection: Any adapter, connection = open_legacy_warehouse( project, test_case.adapter_name, test_case.connection_key ) - append_audit_history(adapter, connection, fixture) - for column in ("audit_description", "audit_meta_json"): - _ = adapter.execute( - connection=connection, sql=f"ALTER TABLE _sqlbuild_audit_results DROP COLUMN {column}" - ) + test_case.append_history(adapter, connection, fixture) + table: str = f"_sqlbuild_{fixture['kind']}" + source_key: str = fixture["source_key"] + for column in test_case.legacy_columns: + _ = adapter.execute(connection=connection, sql=f"ALTER TABLE {table} DROP COLUMN {column}") adapter.close(connection) - pages: list[dict[str, Any]] = warehouse_history_pages(project, "audit_results") + pages: list[dict[str, Any]] = warehouse_history_pages(project, fixture["kind"]) rows: list[dict[str, Any]] = [] for page in pages: rows.extend(page["records"]) - assert [len(page["records"]) for page in pages] == [100, 100, 4] - assert {row["result_id"] for row in rows} == { - f"result-{index:03}" for index in range(fixture["source_rows"]) - } - expected: dict[str, Any] = fixture["record"] | { - "audit_description": None, - "audit_meta_json": None, + assert [len(page["records"]) for page in pages] == fixture["page_lengths"] + assert {row[source_key] for row in rows} == { + f"{fixture['id_prefix']}-{index:03}" for index in range(fixture["source_rows"]) } + expected: dict[str, Any] = fixture["record"] | dict.fromkeys(test_case.legacy_columns) assert rows[0] == expected assert rows[0]["run_id"] == test_case.expected_run_id - assert rows[100] == expected | {"occurred_at": fixture["later_time"]} - assert rows[-1] == expected | {"result_id": fixture["retained_result"]} + assert rows[fixture["later_index"]] == expected | fixture["later_record"] + assert rows[-1] == expected | fixture["last_record"] recent_pages: list[dict[str, Any]] = warehouse_history_pages( - project, "audit_results", fixture["later_time"] + project, fixture["kind"], fixture["later_time"] ) recent_rows: list[dict[str, Any]] = [] for page in recent_pages: recent_rows.extend(page["records"]) - assert recent_rows == [expected | {"occurred_at": fixture["later_time"]}] + assert recent_rows == [expected | fixture["later_record"]] connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) columns: set[str] = { column.name for column in adapter.get_columns( - connection=connection, database=None, schema="main", name="_sqlbuild_audit_results" + connection=connection, database=None, schema="main", name=table ) } - assert not columns & {"audit_description", "audit_meta_json"}, ( + assert not columns.intersection(test_case.legacy_columns), ( "the reader never upgrades source storage" ) assert adapter.query( - connection=connection, sql="SELECT COUNT(*) FROM _sqlbuild_audit_results", limit=1 - ).rows == ((fixture["source_rows"] + fixture["duplicate_rows"] + 1,),) + connection=connection, sql=f"SELECT COUNT(*) FROM {table}", limit=1 + ).rows == ((fixture["source_count"],),) adapter.close(connection) assert not (project / ".sqlbuild/control.sqlite3").exists() @@ -127,24 +162,60 @@ def test_given_external_audit_results_when_reading_legacy_pages_then_duplicates_ @pytest.mark.parametrize( "test_case", [ - CatalogConnectionTestCase("DuckDB automatic audit mirror", "duckdb", "database", ""), - CatalogConnectionTestCase("SQLite automatic audit mirror", "sqlite", "path", ""), + StateEventsTestCase( + f"{adapter_name} {fixture_name}", + adapter_name, + connection_key, + expected_run_id, + fixture_name, + append_history, + legacy_columns, + ) + for adapter_name, connection_key, fixture_name, append_history, expected_run_id, legacy_columns in [ + ( + "duckdb", + "database", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ( + "duckdb", + "database", + "janitor_event_history", + append_janitor_history, + "cleanup-run", + (), + ), + ( + "sqlite", + "path", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + ] ], ids=lambda case: case.description, ) -def test_given_a_served_location_when_audit_history_is_rebuilt_and_retained_then_only_complete_scans_delete( - test_case: CatalogConnectionTestCase, tmp_path: Path +def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retained_then_only_complete_scans_delete( + test_case: StateEventsTestCase, tmp_path: Path ) -> None: project: Path = tmp_path / "project" fixture: dict[str, Any] = json.loads( - Path("benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json").read_text() + Path( + f"benchmarks/duckshop/edge_cases/orchestration/{test_case.fixture_name}.json" + ).read_text() ) adapter: Any connection: Any adapter, connection = open_legacy_warehouse( project, test_case.adapter_name, test_case.connection_key ) - append_audit_history(adapter, connection, fixture) + test_case.append_history(adapter, connection, fixture) adapter.close(connection) (project / "orchestration/orders.py").write_text("") write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) @@ -172,14 +243,18 @@ def test_given_a_served_location_when_audit_history_is_rebuilt_and_retained_then ) as location: _ = wait_for_generation(api, "development", 1, location, log) identities: dict[str, str] = {} - all_ids: set[str] = {f"result-{index:03}" for index in range(fixture["source_rows"])} + source_key: str = fixture["source_key"] + mirror_table: str = f"sqb_warehouse_{fixture['kind']}" + all_ids: set[str] = { + f"{fixture['id_prefix']}-{index:03}" for index in range(fixture["source_rows"]) + } for expected_ids, mirror_sql, warehouse_sql in ( - (all_ids, "DELETE FROM sqb_warehouse_audit_results", "SELECT 1"), + (all_ids, f"DELETE FROM {mirror_table}", "SELECT 1"), ( all_ids, "SELECT 1", - "DELETE FROM _sqlbuild_audit_results " - f"WHERE result_id <> '{fixture['retained_result']}'", + f"DELETE FROM _sqlbuild_{fixture['kind']} " + f"WHERE {source_key} <> '{fixture['retained_result']}'", ), ({fixture["retained_result"]}, "SELECT 1", "SELECT 1"), ): @@ -190,15 +265,19 @@ def test_given_a_served_location_when_audit_history_is_rebuilt_and_retained_then assert time.monotonic() < deadline, log.read_text() with closing(sqlite3.connect(control)) as database: mirrored = database.execute( - "SELECT event_id,record_json FROM sqb_warehouse_audit_results ORDER BY event_id" + f"SELECT event_id,record_json FROM {mirror_table} ORDER BY event_id" ).fetchall() time.sleep(0.05) - assert {json.loads(row)["result_id"] for _, row in mirrored} == expected_ids + assert {json.loads(row)[source_key] for _, row in mirrored} == expected_ids for event_id, text in mirrored: record: dict[str, Any] = json.loads(text) - assert record == fixture["record"] | {"result_id": record["result_id"]} + assert record == ( + fixture["record"] + | {source_key: record[source_key]} + | fixture["record_overrides"].get(record[source_key], {}) + ) assert record["run_id"] == test_case.expected_run_id - assert identities.setdefault(record["result_id"], event_id) == event_id + assert identities.setdefault(record[source_key], event_id) == event_id with closing(sqlite3.connect(control)) as database: _ = database.execute(mirror_sql) database.commit() From a0b5bbfe1e45c79f64841414cce351472cd42f4b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:01:11 +0000 Subject: [PATCH 092/138] feat(orchestration): mirror migration and old-name history Preserve immutable model-move and compatibility-view facts through the existing fenced warehouse history pipeline, with nullable legacy field support and real-writer acceptance fixtures. --- .../orchestration/migration_history.json | 37 ++++ .../orchestration/old_name_history.json | 42 ++++ clients/typescript/src/schema.d.ts | 194 +++++++++++++++++- clients/typescript/tests/types.ts | 18 ++ .../src/native_commands/_helpers/execution.rs | 18 ++ .../src/api/_helpers/resources/router.rs | 8 + .../_helpers/resources/warehouse_metadata.rs | 40 +++- .../sqlbuild-orchestration/src/api/models.rs | 2 + .../src/platform/_helpers/work_client.rs | 20 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/warehouse_migrations.sql | 31 +++ .../main/ingest_migrations.rs | 74 +++++++ .../main/ingest_old_name_views.rs | 79 +++++++ .../src/warehouse_metadata/main/mod.rs | 2 + .../src/warehouse_metadata/models.rs | 85 +++++++- .../tests/api/test_warehouse_metadata.rs | 10 + .../api/warehouse_metadata/events/helpers.rs | 3 +- .../tests/credentials/test_audit.rs | 3 + .../tests/store/helpers.rs | 3 + .../native_host/warehouse_metadata.rs | 68 +++++- fensu.toml | 2 +- .../compiled_project/publication/helpers.py | 96 ++++++++- .../publication/test_warehouse_metadata.py | 73 ++++++- 24 files changed, 884 insertions(+), 30 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/migration_history.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/old_name_history.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_migrations.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/migration_history.json b/benchmarks/duckshop/edge_cases/orchestration/migration_history.json new file mode 100644 index 0000000000..e51a92592c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/migration_history.json @@ -0,0 +1,37 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "move-000", + "target_name": null, + "origin_model": null, + "origin_database": null, + "origin_schema": "legacy", + "origin_name": "orders_🧾", + "destination_model": "orders", + "destination_database": null, + "destination_schema": "main", + "destination_name": "orders_current", + "origin_version_hash": "orders-version", + "discovery": "manual", + "decision": "migrate", + "run_id": "external-move", + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "move-103", + "kind": "migrations", + "source_key": "event_id", + "timestamp_key": "created_at", + "legacy_columns": ["target_name", "origin_model"], + "id_prefix": "move", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": {"event_id": "move-103", "created_at": "2026-02-03T04:05:06.000001Z"}, + "last_record": {"event_id": "move-103", "created_at": "2026-02-03T04:05:06.000001Z"}, + "record_overrides": {"move-103": {"created_at": "2026-02-03T04:05:06.000001Z"}}, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/old_name_history.json b/benchmarks/duckshop/edge_cases/orchestration/old_name_history.json new file mode 100644 index 0000000000..13d8a8496b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/old_name_history.json @@ -0,0 +1,42 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "view-000", + "target_name": null, + "event_type": "required", + "migration_event_id": "move-000", + "destination_model": "orders", + "old_database": null, + "old_schema": "legacy", + "old_name": "orders_🧾", + "new_database": null, + "new_schema": "main", + "new_name": "orders_current", + "view_retention": "7d", + "archive_name": null, + "column_aliases": null, + "grants_copied": null, + "view_sql": null, + "expires_at": null, + "drop_reason": null, + "run_id": "external-move", + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "view-103", + "kind": "old_name_views", + "source_key": "event_id", + "timestamp_key": "created_at", + "legacy_columns": ["view_retention"], + "id_prefix": "view", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": {"event_id": "view-103", "created_at": "2026-02-03T04:05:06.000001Z", "column_aliases": "{\"order_id\":\"legacy_order_id\"}", "expires_at": "2026-02-10T04:05:06Z", "view_sql": "SELECT 'o''rders 🧾' AS label"}, + "last_record": {"event_id": "view-103", "created_at": "2026-02-03T04:05:06.000001Z", "column_aliases": "{\"order_id\":\"legacy_order_id\"}", "expires_at": "2026-02-10T04:05:06Z", "view_sql": "SELECT 'o''rders 🧾' AS label"}, + "record_overrides": {"view-103": {"created_at": "2026-02-03T04:05:06.000001Z", "column_aliases": "{\"order_id\":\"legacy_order_id\"}", "expires_at": "2026-02-10T04:05:06Z", "view_sql": "SELECT 'o''rders 🧾' AS label"}}, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 390b83ee51..01360caed1 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -323,6 +323,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/migrations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_migrations"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata/old-name-views": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_old_name_views"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/source-freshness": { parameters: { query?: never; @@ -855,6 +887,14 @@ export interface components { /** Format: uuid */ session_id: string; }; + MigrationsBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseMigrationRecord"][]; + /** Format: uuid */ + session_id: string; + }; MutationResponse: { applied: boolean; }; @@ -862,6 +902,14 @@ export interface components { NodeEventKind: "success" | "failed" | "recovered"; /** @enum {string} */ NodeOutcome: "success" | "warn" | "failed" | "skipped" | "cancelled"; + OldNameViewsBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseOldNameRecord"][]; + /** Format: uuid */ + session_id: string; + }; /** @description Output of one stream that was not recorded because it exceeded `max_output_bytes`. */ OmittedOutput: { /** Format: int64 */ @@ -1455,7 +1503,7 @@ export interface components { * @description Only immutable history tables support a timestamp watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -1477,6 +1525,54 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Model-move history is evidence, not authority to migrate a warehouse relation. */ + WarehouseMigrationRecord: { + /** Format: date-time */ + created_at: string; + decision: string; + destination_database?: string | null; + destination_model: string; + destination_name: string; + destination_schema: string; + discovery: string; + event_id: string; + origin_database?: string | null; + origin_model?: string | null; + origin_name: string; + origin_schema?: string | null; + origin_version_hash: string; + run_id: string; + state_database?: string | null; + state_schema: string; + target_name?: string | null; + }; + /** @description Preserve nullable compatibility-view facts without projecting live warehouse existence. */ + WarehouseOldNameRecord: { + archive_name?: string | null; + column_aliases?: string | null; + /** Format: date-time */ + created_at: string; + destination_model: string; + drop_reason?: string | null; + event_id: string; + event_type: string; + /** Format: date-time */ + expires_at?: string | null; + grants_copied?: string | null; + migration_event_id: string; + new_database?: string | null; + new_name: string; + new_schema: string; + old_database?: string | null; + old_name: string; + old_schema?: string | null; + run_id: string; + state_database?: string | null; + state_schema: string; + target_name?: string | null; + view_retention?: string | null; + view_sql?: string | null; + }; WarehouseReconciliationCursor: { /** Format: uuid */ event_id: string; @@ -2341,6 +2437,102 @@ export interface operations { }; }; }; + report_migrations: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MigrationsBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_old_name_views: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["OldNameViewsBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_source_freshness: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 612bfb4953..05700417f4 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -39,6 +39,24 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/janitor params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const migrationHistory: components["schemas"]["WarehouseHistoryKind"] = "migrations"; +const oldNameHistory: components["schemas"]["WarehouseHistoryKind"] = "old_name_views"; +const optionalExpiry: components["schemas"]["WarehouseOldNameRecord"]["expires_at"] = null; +const optionalAliases: components["schemas"]["WarehouseOldNameRecord"]["column_aliases"] = null; +const createdAt: components["schemas"]["WarehouseMigrationRecord"]["created_at"] = "2026-01-02T03:04:05Z"; +void migrationHistory; +void oldNameHistory; +void optionalExpiry; +void optionalAliases; +void createdAt; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/migrations", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/old-name-views", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 0093697096..f4e3e2d774 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -433,6 +433,24 @@ pub(crate) async fn deliver_metadata_page( ) .await } + WarehouseHistoryKind::Migrations => { + client + .sync_migrations( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid migrations batch"))?, + ) + .await + } + WarehouseHistoryKind::OldNameViews => { + client + .sync_old_name_views( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid old name views batch"))?, + ) + .await + } } } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index f59703b526..f91dbc582f 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -47,6 +47,14 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_migrations(&store, (location_id, &target), &batch).await?, + )) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/old-name-views", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = OldNameViewsBatch, + responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report_old_name_views( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_old_name_views(&store, (location_id, &target), &batch).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 039f320df3..bfa687b93a 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -52,6 +52,8 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report_fingerprints, warehouse_metadata::report_audit_results, warehouse_metadata::report_janitor_events, + warehouse_metadata::report_migrations, + warehouse_metadata::report_old_name_views, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index f2d57ba30e..25985f3d88 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,8 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - AuditResultsBatch, FingerprintBatch, JanitorEventsBatch, SourceFreshnessBatch, - WarehouseSyncResponse, WarehouseWatermarkRead, + AuditResultsBatch, FingerprintBatch, JanitorEventsBatch, MigrationsBatch, OldNameViewsBatch, + SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -59,6 +59,22 @@ impl LocationWorkClient { self.sync_metadata((target, "janitor-events"), batch).await } + pub async fn sync_migrations( + &self, + target: &str, + batch: &MigrationsBatch, + ) -> Result { + self.sync_metadata((target, "migrations"), batch).await + } + + pub async fn sync_old_name_views( + &self, + target: &str, + batch: &OldNameViewsBatch, + ) -> Result { + self.sync_metadata((target, "old-name-views"), batch).await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index c7ca01f70a..3041bc9518 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -29,6 +29,7 @@ pub(crate) const WAREHOUSE_RECONCILIATION: &str = include_str!("../warehouse_rec pub(crate) const AUDIT_STORAGE_ORDER: &str = include_str!("../audit_storage_order.sql"); pub(crate) const WAREHOUSE_AUDIT_RESULTS: &str = include_str!("../warehouse_audit_results.sql"); pub(crate) const WAREHOUSE_JANITOR_EVENTS: &str = include_str!("../warehouse_janitor_events.sql"); +pub(crate) const WAREHOUSE_MIGRATIONS: &str = include_str!("../warehouse_migrations.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 1e9b1ec8a0..50bde54ef1 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -8,8 +8,8 @@ use crate::store::_helpers::migration::{ DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_RECONCILIATION, render, - verify_migrations, + WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, + render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -54,6 +54,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("audit_storage_order", AUDIT_STORAGE_ORDER), ("warehouse_audit_results", WAREHOUSE_AUDIT_RESULTS), ("warehouse_janitor_events", WAREHOUSE_JANITOR_EVENTS), + ("warehouse_migrations", WAREHOUSE_MIGRATIONS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_migrations.sql b/crates/sqlbuild-orchestration/src/store/warehouse_migrations.sql new file mode 100644 index 0000000000..af36394ad9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_migrations.sql @@ -0,0 +1,31 @@ +CREATE TABLE sqb_warehouse_migrations ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + destination_model TEXT NOT NULL, + decision TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_migrations_reconciliation ON sqb_warehouse_migrations (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_old_name_views ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + migration_event_id TEXT NOT NULL, + destination_model TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_old_name_views_reconciliation ON sqb_warehouse_old_name_views (location_id,target,state_database,state_schema,observed_at,event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs new file mode 100644 index 0000000000..f6400a22e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs @@ -0,0 +1,74 @@ +//! Mirror model-move facts without changing warehouse migration ownership. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + MigrationsBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, +}; + +pub async fn ingest_migrations( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MigrationsBatch, +) -> Result { + validate_fields( + ("migrations", batch.generation, batch.records.len()), + false, + false, + )?; + ingest_events( + store, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::Migrations, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.destination_model, + &record.destination_schema, + &record.destination_name, + ] + }) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: vec![ + ( + "target_name", + Parameter::OptionalText(record.target_name.clone()), + ), + ( + "destination_model", + Parameter::Text(record.destination_model.clone()), + ), + ("decision", Parameter::Text(record.decision.clone())), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs new file mode 100644 index 0000000000..082e75f161 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs @@ -0,0 +1,79 @@ +//! Mirror compatibility-view history, never an assertion about current relation existence. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + OldNameViewsBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, +}; + +pub async fn ingest_old_name_views( + store: &ControlStore, + scope: (Uuid, &str), + batch: &OldNameViewsBatch, +) -> Result { + validate_fields( + ("old name views", batch.generation, batch.records.len()), + false, + false, + )?; + ingest_events( + store, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::OldNameViews, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.migration_event_id, + &record.destination_model, + &record.new_schema, + &record.new_name, + ] + }) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: vec![ + ( + "target_name", + Parameter::OptionalText(record.target_name.clone()), + ), + ( + "migration_event_id", + Parameter::Text(record.migration_event_id.clone()), + ), + ( + "destination_model", + Parameter::Text(record.destination_model.clone()), + ), + ("event_type", Parameter::Text(record.event_type.clone())), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index a9cd5bb8a7..450d9904b4 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,5 +1,7 @@ pub mod ingest_audit_results; pub mod ingest_fingerprints; pub mod ingest_janitor_events; +pub mod ingest_migrations; +pub mod ingest_old_name_views; pub mod ingest_source_freshness; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 4ef54db78c..022c347b0b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -123,6 +123,77 @@ pub struct JanitorEventsBatch { pub reconciliation: Option, } +/// Model-move history is evidence, not authority to migrate a warehouse relation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseMigrationRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub target_name: Option, + pub origin_model: Option, + pub origin_database: Option, + pub origin_schema: Option, + pub origin_name: String, + pub destination_model: String, + pub destination_database: Option, + pub destination_schema: String, + pub destination_name: String, + pub origin_version_hash: String, + pub discovery: String, + pub decision: String, + pub run_id: String, + pub created_at: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct MigrationsBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + +/// Preserve nullable compatibility-view facts without projecting live warehouse existence. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseOldNameRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub target_name: Option, + pub event_type: String, + pub migration_event_id: String, + pub destination_model: String, + pub old_database: Option, + pub old_schema: Option, + pub old_name: String, + pub new_database: Option, + pub new_schema: String, + pub new_name: String, + pub view_retention: Option, + pub archive_name: Option, + pub column_aliases: Option, + pub grants_copied: Option, + pub view_sql: Option, + pub expires_at: Option>, + pub drop_reason: Option, + pub run_id: String, + pub created_at: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct OldNameViewsBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + /// Physical namespace of SQLBuild's source-freshness state table, never a source query. #[derive( Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, @@ -141,14 +212,18 @@ pub enum WarehouseHistoryKind { SourceFreshness, AuditResults, JanitorEvents, + Migrations, + OldNameViews, } impl WarehouseHistoryKind { - pub const ALL: [Self; 4] = [ + pub const ALL: [Self; 6] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, Self::JanitorEvents, + Self::Migrations, + Self::OldNameViews, ]; #[must_use] @@ -158,6 +233,8 @@ impl WarehouseHistoryKind { Self::SourceFreshness => "source_freshness", Self::AuditResults => "audit_results", Self::JanitorEvents => "janitor_events", + Self::Migrations => "migrations", + Self::OldNameViews => "old_name_views", } } @@ -168,6 +245,8 @@ impl WarehouseHistoryKind { Self::SourceFreshness => "sqb_warehouse_freshness", Self::AuditResults => "sqb_warehouse_audit_results", Self::JanitorEvents => "sqb_warehouse_janitor_events", + Self::Migrations => "sqb_warehouse_migrations", + Self::OldNameViews => "sqb_warehouse_old_name_views", } } } @@ -280,6 +359,8 @@ pub enum WarehouseHistoryRead { Fingerprints(FingerprintRead), AuditResults(StateEventsRead), JanitorEvents(StateEventsRead), + Migrations(StateEventsRead), + OldNameViews(StateEventsRead), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -289,6 +370,8 @@ pub enum WarehouseHistoryRecord { Fingerprint(WarehouseFingerprintRecord), AuditResult(Box), JanitorEvent(WarehouseJanitorRecord), + Migration(Box), + OldNameView(Box), } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index e77332366b..0f6d0b1505 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -333,6 +333,16 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "janitor_event_history", "event_id", ), + ( + WarehouseHistoryKind::Migrations, + "migration_history", + "event_id", + ), + ( + WarehouseHistoryKind::OldNameViews, + "old_name_history", + "event_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs index d5defa43b2..3c02ee4ff4 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -31,6 +31,7 @@ pub(crate) async fn event_history( )); let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) .expect("event history"); + let timestamp = fixture["timestamp_key"].as_str().unwrap_or("occurred_at"); let record = fixture["record"].clone(); let batch = json!({"session_id": session_id, "generation": 1, "records": [record, record]}); let url = freshness_url.replace("source-freshness", &kind.as_str().replace('_', "-")); @@ -129,7 +130,7 @@ pub(crate) async fn event_history( .await .expect("watermark") ), - record["occurred_at"] + record[timestamp] ); let fault = TransactionFault { description: "event page rolls back atomically", diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index c5ae497678..330cb2644d 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -124,6 +124,9 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_old_name_views", + "DROP TABLE sqb_warehouse_migrations", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_migrations'", "DROP TABLE sqb_warehouse_janitor_events", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_janitor_events'", "DROP TABLE sqb_warehouse_audit_results", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index c3312cc506..2aeb88cd39 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,9 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_old_name_views", + "DROP TABLE sqb_warehouse_migrations", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_migrations'", "DROP TABLE sqb_warehouse_janitor_events", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_janitor_events'", "DROP TABLE sqb_warehouse_audit_results", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index a772cf1e73..01812c38b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -14,6 +14,7 @@ use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, }; +use sqlbuild_state::events::constants::{MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS}; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; @@ -41,9 +42,10 @@ pub(crate) fn read_history( let location = match request { WarehouseHistoryRead::SourceFreshness(read) => &read.location, WarehouseHistoryRead::Fingerprints(read) => &read.location, - WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::JanitorEvents(read) => { - &read.location - } + WarehouseHistoryRead::AuditResults(read) + | WarehouseHistoryRead::JanitorEvents(read) + | WarehouseHistoryRead::Migrations(read) + | WarehouseHistoryRead::OldNameViews(read) => &read.location, }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -152,6 +154,24 @@ fn read_page( event_columns.as_slice(), "occurred_at", ), + WarehouseHistoryRead::Migrations(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_migrations", + MIGRATION_COLUMNS.as_slice(), + "created_at", + ), + WarehouseHistoryRead::OldNameViews(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_old_name_views", + OLD_NAME_VIEW_COLUMNS.as_slice(), + "created_at", + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -195,10 +215,36 @@ fn read_page( actual_columns.insert(field(&column?, "name")?.extract::()?.to_lowercase()); } let has_event_id = actual_columns.contains("event_id"); - let audit = matches!(request, WarehouseHistoryRead::AuditResults(_)); + let optional_columns: &[&str] = match request { + WarehouseHistoryRead::AuditResults(_) => &["audit_description", "audit_meta_json"], + WarehouseHistoryRead::Migrations(_) => &[ + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "destination_database", + ], + WarehouseHistoryRead::OldNameViews(_) => &[ + "target_name", + "old_database", + "old_schema", + "new_database", + "view_retention", + "archive_name", + "column_aliases", + "grants_copied", + "view_sql", + "expires_at", + "drop_reason", + ], + _ => &[], + }; let key_columns: Option<&[&str]> = match request { WarehouseHistoryRead::AuditResults(_) => Some(&["result_id", "occurred_at"]), WarehouseHistoryRead::JanitorEvents(_) => Some(&["event_id", "occurred_at"]), + WarehouseHistoryRead::Migrations(_) | WarehouseHistoryRead::OldNameViews(_) => { + Some(&["event_id", "created_at"]) + } _ => None, }; let mut cursor_indexes: Vec = columns @@ -257,10 +303,7 @@ fn read_page( let projection: Vec = columns .iter() .map(|column| { - if audit - && matches!(*column, "audit_description" | "audit_meta_json") - && !actual_columns.contains(*column) - { + if optional_columns.contains(column) && !actual_columns.contains(*column) { format!("NULL AS {column}") } else { (*column).to_owned() @@ -314,6 +357,8 @@ fn read_page( WarehouseHistoryRecord::Fingerprint(record) => record.ts, WarehouseHistoryRecord::AuditResult(record) => record.occurred_at, WarehouseHistoryRecord::JanitorEvent(record) => record.occurred_at, + WarehouseHistoryRecord::Migration(record) => record.created_at, + WarehouseHistoryRecord::OldNameView(record) => record.created_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -347,7 +392,7 @@ fn decode_record( qualified, )?, ), - WarehouseHistoryRead::AuditResults(read) => { + WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } WarehouseHistoryRead::JanitorEvents(read) => { @@ -357,6 +402,9 @@ fn decode_record( (columns, &[timestamp, "archived_at"]), ); } + WarehouseHistoryRead::OldNameViews(read) => { + return decode_event_record(&read.location, row, (columns, &[timestamp, "expires_at"])); + } }; let mut value = json!({ "state_database": location.database, @@ -398,7 +446,7 @@ fn decode_event_record( let mut value = json!({"state_database": location.database, "state_schema": location.schema}); for (index, name) in columns.iter().enumerate() { let item = row.get_item(index)?; - value[*name] = if timestamps.contains(name) { + value[*name] = if timestamps.contains(name) && !item.is_none() { let item = if item.is_instance_of::() { item.py() .import("datetime")? diff --git a/fensu.toml b/fensu.toml index 95e23d4d31..8181b019b1 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1662,7 +1662,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 5fc97b3d4f..ff26bb9750 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta from io import StringIO from pathlib import Path -from typing import Any, cast +from typing import Any, Literal, cast from urllib.error import HTTPError from urllib.parse import quote from urllib.request import Request, urlopen @@ -26,7 +26,17 @@ import duckdb import pytest -from sqlbuild._native import Fingerprint, write_fingerprint +from sqlbuild._native import ( + Fingerprint, + MigrationDecision, + MigrationDiscovery, + MigrationEvent, + MigrationRelation, + OldNameViewEvent, + OldNameViewEventType, + write_fingerprint, + write_migration_state_event, +) from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.executor.audit_results.main._write import write_audit_result_records from sqlbuild.executor.audit_results.models import AuditResultRecord @@ -180,6 +190,88 @@ def append_janitor_history(adapter: Any, connection: Any, fixture: dict[str, Any writer.write(connection=connection, records=(records[0],) * fixture["duplicate_rows"]) +def append_migration_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real model-move facts, including idempotent duplicate attempts.""" + data: dict[str, Any] = migration_history_data(fixture, ("origin", "destination")) + data["discovery"] = MigrationDiscovery(data["discovery"]) + data["decision"] = MigrationDecision(data["decision"]) + record: MigrationEvent = MigrationEvent(**data) + write_migration_history( + adapter, + connection, + fixture, + record, + "model", + adapter.render_create_migration_state_table_sql(database=None, schema="main"), + {}, + ) + + +def append_old_name_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real compatibility-view facts, including nullable and populated expiry.""" + data: dict[str, Any] = migration_history_data(fixture, ("old", "new")) + data["event_type"] = OldNameViewEventType(data["event_type"]) + data["column_aliases"] = () + record: OldNameViewEvent = OldNameViewEvent(**data) + write_migration_history( + adapter, + connection, + fixture, + record, + "old_name", + adapter.render_create_old_name_view_state_table_sql(database=None, schema="main"), + { + "column_aliases": tuple(json.loads(fixture["last_record"]["column_aliases"]).items()), + "expires_at": datetime.fromisoformat(fixture["last_record"]["expires_at"]), + "view_sql": fixture["last_record"]["view_sql"], + }, + ) + + +def migration_history_data(fixture: dict[str, Any], relations: tuple[str, str]) -> dict[str, Any]: + """Reconstruct native relation values from the warehouse-shaped fixture.""" + data: dict[str, Any] = fixture["record"].copy() + del data["state_database"] + del data["state_schema"] + data["created_at"] = datetime.fromisoformat(data["created_at"]) + for relation in relations: + data[relation] = MigrationRelation( + database=data.pop(f"{relation}_database"), + schema=data.pop(f"{relation}_schema"), + name=data.pop(f"{relation}_name"), + ) + return data + + +def write_migration_history( + adapter: Any, + connection: Any, + fixture: dict[str, Any], + record: MigrationEvent | OldNameViewEvent, + kind: Literal["model", "old_name"], + create_sql: str, + last_values: dict[str, Any], +) -> None: + """Exercise the shared immutable writer with paginated, retried source facts.""" + records: list[Any] = [ + replace(record, event_id=f"{fixture['id_prefix']}-{index:03}") + for index in range(fixture["source_rows"]) + ] + records[-1] = replace( + records[-1], created_at=datetime.fromisoformat(fixture["later_time"]), **last_values + ) + records.extend([records[0]] * fixture["duplicate_rows"]) + for record in records: + write_migration_state_event( + kind=kind, + connection=connection, + execute=adapter.execute, + event=record, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=create_sql, + ) + + def warehouse_history_page( project: Path, table: str, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index b6dfb5207c..ab211ffd66 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -47,6 +47,8 @@ append_contended_fingerprints, append_janitor_history, append_legacy_history, + append_migration_history, + append_old_name_history, append_retried_fingerprints, location_process, open_legacy_warehouse, @@ -98,6 +100,40 @@ ("audit_description", "audit_meta_json"), ), ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + ( + "duckdb", + "database", + "migration_history", + append_migration_history, + "external-move", + (), + ), + ( + "duckdb", + "database", + "old_name_history", + append_old_name_history, + "external-move", + (), + ), + ("sqlite", "path", "migration_history", append_migration_history, "external-move", ()), + ("sqlite", "path", "old_name_history", append_old_name_history, "external-move", ()), + ( + "duckdb", + "database", + "old_name_history", + append_old_name_history, + "external-move", + ("column_aliases",), + ), + ( + "sqlite", + "path", + "old_name_history", + append_old_name_history, + "external-move", + ("column_aliases",), + ), ] ], ids=lambda case: case.description, @@ -119,7 +155,10 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and test_case.append_history(adapter, connection, fixture) table: str = f"_sqlbuild_{fixture['kind']}" source_key: str = fixture["source_key"] - for column in test_case.legacy_columns: + legacy_columns: tuple[str, ...] = test_case.legacy_columns + tuple( + fixture.get("legacy_columns", ()) + ) + for column in legacy_columns: _ = adapter.execute(connection=connection, sql=f"ALTER TABLE {table} DROP COLUMN {column}") adapter.close(connection) pages: list[dict[str, Any]] = warehouse_history_pages(project, fixture["kind"]) @@ -130,18 +169,20 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and assert {row[source_key] for row in rows} == { f"{fixture['id_prefix']}-{index:03}" for index in range(fixture["source_rows"]) } - expected: dict[str, Any] = fixture["record"] | dict.fromkeys(test_case.legacy_columns) + expected: dict[str, Any] = fixture["record"] | dict.fromkeys(legacy_columns) assert rows[0] == expected assert rows[0]["run_id"] == test_case.expected_run_id - assert rows[fixture["later_index"]] == expected | fixture["later_record"] - assert rows[-1] == expected | fixture["last_record"] + assert rows[fixture["later_index"]] == ( + expected | fixture["later_record"] | dict.fromkeys(legacy_columns) + ) + assert rows[-1] == expected | fixture["last_record"] | dict.fromkeys(legacy_columns) recent_pages: list[dict[str, Any]] = warehouse_history_pages( project, fixture["kind"], fixture["later_time"] ) recent_rows: list[dict[str, Any]] = [] for page in recent_pages: recent_rows.extend(page["records"]) - assert recent_rows == [expected | fixture["later_record"]] + assert recent_rows == [expected | fixture["later_record"] | dict.fromkeys(legacy_columns)] connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) columns: set[str] = { column.name @@ -149,9 +190,7 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and connection=connection, database=None, schema="main", name=table ) } - assert not columns.intersection(test_case.legacy_columns), ( - "the reader never upgrades source storage" - ) + assert not columns.intersection(legacy_columns), "the reader never upgrades source storage" assert adapter.query( connection=connection, sql=f"SELECT COUNT(*) FROM {table}", limit=1 ).rows == ((fixture["source_count"],),) @@ -197,6 +236,24 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and ("audit_description", "audit_meta_json"), ), ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + ( + "duckdb", + "database", + "migration_history", + append_migration_history, + "external-move", + (), + ), + ( + "duckdb", + "database", + "old_name_history", + append_old_name_history, + "external-move", + (), + ), + ("sqlite", "path", "migration_history", append_migration_history, "external-move", ()), + ("sqlite", "path", "old_name_history", append_old_name_history, "external-move", ()), ] ], ids=lambda case: case.description, From ef8a0731340cd15b2689f8f17620aa250352f33c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:51:44 +0000 Subject: [PATCH 093/138] feat: mirror immutable microbatch warehouse history --- .../orchestration/microbatch_history.json | 109 +++++++++++++++ clients/typescript/src/schema.d.ts | 124 +++++++++++++++++- clients/typescript/tests/types.ts | 10 ++ .../src/native_commands/_helpers/execution.rs | 9 ++ .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 22 +++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 12 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/warehouse_microbatches.sql | 21 +++ .../main/ingest_microbatches.rs | 89 +++++++++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 64 ++++++++- .../tests/api/test_warehouse_metadata.rs | 5 + .../tests/credentials/test_audit.rs | 2 + .../tests/store/helpers.rs | 2 + .../native_host/warehouse_metadata.rs | 39 +++++- fensu.toml | 2 +- .../compiled_project/publication/helpers.py | 25 ++++ .../publication/test_warehouse_metadata.py | 8 +- 21 files changed, 540 insertions(+), 15 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_microbatches.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json b/benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json new file mode 100644 index 0000000000..9f3464975e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json @@ -0,0 +1,109 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "batch-000", + "record_type": "partition_completion", + "scope_kind": "direct_logical", + "scope_key": "orders:main.orders_🧾", + "model_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders_🧾", + "physical_generation_id": "orders-generation-1", + "origin_run_id": "external-origin", + "origin_run_started_at": "2026-02-02T03:04:05Z", + "execution_run_id": "", + "execution_run_started_at": null, + "run_type": "normal", + "completion_type": "initial", + "run_start": "18446744073709551616", + "run_end": "18446744073709551627", + "partition_start": "18446744073709551616", + "partition_end": "18446744073709551619", + "batch_size": "3", + "cursor_column": "order_id", + "cursor_type": "integer", + "cursor_grain": null, + "model_version_hash": "orders-version", + "definition_hash": "orders-definition", + "fingerprint_status": "known", + "replay_requirement_id": null, + "required_model_version_hash": null, + "previous_model_version_hash": null, + "replay_policy": null, + "rows_affected": 7, + "completed_at": "2026-02-03T03:04:05Z", + "coverage_source": null, + "observed_row_count": null, + "observed_at": "2026-02-03T05:06:07Z", + "synthetic_reason": null, + "unaccounted_policy": null, + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "batch-103", + "kind": "microbatches", + "source_key": "event_id", + "timestamp_key": "created_at", + "run_key": "execution_run_id", + "id_prefix": "batch", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": { + "event_id": "batch-103", + "record_type": "replay_requirement", + "created_at": "2026-02-03T04:05:06.000001Z", + "completion_type": null, + "partition_start": null, + "partition_end": null, + "replay_requirement_id": "requirement-🧾", + "required_model_version_hash": "orders-next-version", + "previous_model_version_hash": "orders-version", + "replay_policy": "all", + "completed_at": null, + "rows_affected": null + }, + "last_record": { + "event_id": "batch-103", + "record_type": "replay_requirement", + "created_at": "2026-02-03T04:05:06.000001Z", + "completion_type": null, + "partition_start": null, + "partition_end": null, + "replay_requirement_id": "requirement-🧾", + "required_model_version_hash": "orders-next-version", + "previous_model_version_hash": "orders-version", + "replay_policy": "all", + "completed_at": null, + "rows_affected": null + }, + "record_overrides": { + "batch-102": { + "record_type": "synthetic_completion", + "fingerprint_status": "unknown", + "model_version_hash": null, + "definition_hash": null, + "observed_row_count": 19, + "synthetic_reason": "Retained 'orders' 🧾", + "unaccounted_policy": "synthesize" + }, + "batch-103": { + "record_type": "replay_requirement", + "created_at": "2026-02-03T04:05:06.000001Z", + "completion_type": null, + "partition_start": null, + "partition_end": null, + "replay_requirement_id": "requirement-🧾", + "required_model_version_hash": "orders-next-version", + "previous_model_version_hash": "orders-version", + "replay_policy": "all", + "completed_at": null, + "rows_affected": null + } + }, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 01360caed1..df3974d453 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -323,6 +323,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/microbatches": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_microbatches"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/migrations": { parameters: { query?: never; @@ -887,6 +903,14 @@ export interface components { /** Format: uuid */ session_id: string; }; + MicrobatchesBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseMicrobatchRecord"][]; + /** Format: uuid */ + session_id: string; + }; MigrationsBatch: { /** Format: int64 */ generation: number; @@ -1503,7 +1527,7 @@ export interface components { * @description Only immutable history tables support a timestamp watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -1525,6 +1549,56 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Immutable coordination evidence, never authority to execute or mark a batch complete. */ + WarehouseMicrobatchRecord: { + batch_size: string; + /** Format: date-time */ + completed_at?: string | null; + completion_type?: string | null; + coverage_source?: string | null; + /** Format: date-time */ + created_at: string; + cursor_column: string; + cursor_grain?: string | null; + cursor_type: string; + definition_hash?: string | null; + event_id: string; + execution_run_id: string; + /** Format: date-time */ + execution_run_started_at?: string | null; + fingerprint_status: string; + model_name: string; + model_version_hash?: string | null; + /** Format: date-time */ + observed_at?: string | null; + /** Format: int64 */ + observed_row_count?: number | null; + origin_run_id: string; + /** Format: date-time */ + origin_run_started_at?: string | null; + partition_end?: string | null; + partition_start?: string | null; + physical_generation_id: string; + previous_model_version_hash?: string | null; + record_type: string; + replay_policy?: string | null; + replay_requirement_id?: string | null; + required_model_version_hash?: string | null; + /** Format: int64 */ + rows_affected?: number | null; + run_end: string; + run_start: string; + run_type: string; + scope_key: string; + scope_kind: string; + state_database?: string | null; + state_schema: string; + synthetic_reason?: string | null; + target_database?: string | null; + target_name: string; + target_schema?: string | null; + unaccounted_policy?: string | null; + }; /** @description Model-move history is evidence, not authority to migrate a warehouse relation. */ WarehouseMigrationRecord: { /** Format: date-time */ @@ -2437,6 +2511,54 @@ export interface operations { }; }; }; + report_microbatches: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MicrobatchesBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_migrations: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 05700417f4..87991ccd5c 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -57,6 +57,16 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/old-nam params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const microbatchHistory: components["schemas"]["WarehouseHistoryKind"] = "microbatches"; +const cursorBound: components["schemas"]["WarehouseMicrobatchRecord"]["run_end"] = "18446744073709551617"; +const optionalCompletion: components["schemas"]["WarehouseMicrobatchRecord"]["completed_at"] = null; +void microbatchHistory; +void cursorBound; +void optionalCompletion; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/microbatches", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index f4e3e2d774..78b2666cc3 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -451,6 +451,15 @@ pub(crate) async fn deliver_metadata_page( ) .await } + WarehouseHistoryKind::Microbatches => { + client + .sync_microbatches( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid microbatches batch"))?, + ) + .await + } } } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index f91dbc582f..bb7cda21f7 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -55,6 +55,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_microbatches(&store, (location_id, &target), &batch).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index bfa687b93a..1a623f0e0d 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -54,6 +54,7 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report_janitor_events, warehouse_metadata::report_migrations, warehouse_metadata::report_old_name_views, + warehouse_metadata::report_microbatches, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 25985f3d88..e4b218c28e 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,8 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - AuditResultsBatch, FingerprintBatch, JanitorEventsBatch, MigrationsBatch, OldNameViewsBatch, - SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, + AuditResultsBatch, FingerprintBatch, JanitorEventsBatch, MicrobatchesBatch, MigrationsBatch, + OldNameViewsBatch, SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -75,6 +75,14 @@ impl LocationWorkClient { self.sync_metadata((target, "old-name-views"), batch).await } + pub async fn sync_microbatches( + &self, + target: &str, + batch: &MicrobatchesBatch, + ) -> Result { + self.sync_metadata((target, "microbatches"), batch).await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 3041bc9518..a6c5364fbc 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -30,6 +30,7 @@ pub(crate) const AUDIT_STORAGE_ORDER: &str = include_str!("../audit_storage_orde pub(crate) const WAREHOUSE_AUDIT_RESULTS: &str = include_str!("../warehouse_audit_results.sql"); pub(crate) const WAREHOUSE_JANITOR_EVENTS: &str = include_str!("../warehouse_janitor_events.sql"); pub(crate) const WAREHOUSE_MIGRATIONS: &str = include_str!("../warehouse_migrations.sql"); +pub(crate) const WAREHOUSE_MICROBATCHES: &str = include_str!("../warehouse_microbatches.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 50bde54ef1..af405ba7c5 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -8,8 +8,8 @@ use crate::store::_helpers::migration::{ DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, - render, verify_migrations, + WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, + WAREHOUSE_RECONCILIATION, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -55,6 +55,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_audit_results", WAREHOUSE_AUDIT_RESULTS), ("warehouse_janitor_events", WAREHOUSE_JANITOR_EVENTS), ("warehouse_migrations", WAREHOUSE_MIGRATIONS), + ("warehouse_microbatches", WAREHOUSE_MICROBATCHES), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_microbatches.sql b/crates/sqlbuild-orchestration/src/store/warehouse_microbatches.sql new file mode 100644 index 0000000000..4d0af9a5d3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_microbatches.sql @@ -0,0 +1,21 @@ +CREATE TABLE sqb_warehouse_microbatches ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + model_name TEXT NOT NULL, + target_name TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + physical_generation_id TEXT NOT NULL, + record_type TEXT NOT NULL, + origin_run_id TEXT NOT NULL, + execution_run_id TEXT NOT NULL, + replay_requirement_id TEXT, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_microbatches_reconciliation ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs new file mode 100644 index 0000000000..737c772ee8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs @@ -0,0 +1,89 @@ +//! Mirror append-only microbatch facts without reading or changing execution coordination. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + MicrobatchesBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, +}; + +pub async fn ingest_microbatches( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MicrobatchesBatch, +) -> Result { + validate_fields( + ("microbatches", batch.generation, batch.records.len()), + false, + false, + )?; + ingest_events( + store, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::Microbatches, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.scope_kind, + &record.scope_key, + &record.model_name, + &record.target_name, + &record.physical_generation_id, + &record.record_type, + ] + }) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: vec![ + ("model_name", Parameter::Text(record.model_name.clone())), + ("target_name", Parameter::Text(record.target_name.clone())), + ("scope_kind", Parameter::Text(record.scope_kind.clone())), + ("scope_key", Parameter::Text(record.scope_key.clone())), + ( + "physical_generation_id", + Parameter::Text(record.physical_generation_id.clone()), + ), + ("record_type", Parameter::Text(record.record_type.clone())), + ( + "origin_run_id", + Parameter::Text(record.origin_run_id.clone()), + ), + ( + "execution_run_id", + Parameter::Text(record.execution_run_id.clone()), + ), + ( + "replay_requirement_id", + Parameter::OptionalText(record.replay_requirement_id.clone()), + ), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 450d9904b4..b249e5a7f0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,6 +1,7 @@ pub mod ingest_audit_results; pub mod ingest_fingerprints; pub mod ingest_janitor_events; +pub mod ingest_microbatches; pub mod ingest_migrations; pub mod ingest_old_name_views; pub mod ingest_source_freshness; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 022c347b0b..f216ca085a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -194,6 +194,62 @@ pub struct OldNameViewsBatch { pub reconciliation: Option, } +/// Immutable coordination evidence, never authority to execute or mark a batch complete. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseMicrobatchRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub record_type: String, + pub scope_kind: String, + pub scope_key: String, + pub model_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: String, + pub physical_generation_id: String, + pub origin_run_id: String, + pub origin_run_started_at: Option>, + pub execution_run_id: String, + pub execution_run_started_at: Option>, + pub run_type: String, + pub completion_type: Option, + pub run_start: String, + pub run_end: String, + pub partition_start: Option, + pub partition_end: Option, + pub batch_size: String, + pub cursor_column: String, + pub cursor_type: String, + pub cursor_grain: Option, + pub model_version_hash: Option, + pub definition_hash: Option, + pub fingerprint_status: String, + pub replay_requirement_id: Option, + pub required_model_version_hash: Option, + pub previous_model_version_hash: Option, + pub replay_policy: Option, + pub rows_affected: Option, + pub completed_at: Option>, + pub coverage_source: Option, + pub observed_row_count: Option, + pub observed_at: Option>, + pub synthetic_reason: Option, + pub unaccounted_policy: Option, + pub created_at: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct MicrobatchesBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + /// Physical namespace of SQLBuild's source-freshness state table, never a source query. #[derive( Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, @@ -214,16 +270,18 @@ pub enum WarehouseHistoryKind { JanitorEvents, Migrations, OldNameViews, + Microbatches, } impl WarehouseHistoryKind { - pub const ALL: [Self; 6] = [ + pub const ALL: [Self; 7] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, Self::JanitorEvents, Self::Migrations, Self::OldNameViews, + Self::Microbatches, ]; #[must_use] @@ -235,6 +293,7 @@ impl WarehouseHistoryKind { Self::JanitorEvents => "janitor_events", Self::Migrations => "migrations", Self::OldNameViews => "old_name_views", + Self::Microbatches => "microbatches", } } @@ -247,6 +306,7 @@ impl WarehouseHistoryKind { Self::JanitorEvents => "sqb_warehouse_janitor_events", Self::Migrations => "sqb_warehouse_migrations", Self::OldNameViews => "sqb_warehouse_old_name_views", + Self::Microbatches => "sqb_warehouse_microbatches", } } } @@ -361,6 +421,7 @@ pub enum WarehouseHistoryRead { JanitorEvents(StateEventsRead), Migrations(StateEventsRead), OldNameViews(StateEventsRead), + Microbatches(StateEventsRead), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -372,6 +433,7 @@ pub enum WarehouseHistoryRecord { JanitorEvent(WarehouseJanitorRecord), Migration(Box), OldNameView(Box), + Microbatch(Box), } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 0f6d0b1505..6fda83aec2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -343,6 +343,11 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "old_name_history", "event_id", ), + ( + WarehouseHistoryKind::Microbatches, + "microbatch_history", + "event_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 330cb2644d..84d0eea7ab 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -124,6 +124,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_microbatches", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_microbatches'", "DROP TABLE sqb_warehouse_old_name_views", "DROP TABLE sqb_warehouse_migrations", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_migrations'", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 2aeb88cd39..2246ec93e8 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_microbatches", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_microbatches'", "DROP TABLE sqb_warehouse_old_name_views", "DROP TABLE sqb_warehouse_migrations", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_migrations'", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 01812c38b6..cb6d20fafc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -45,7 +45,8 @@ pub(crate) fn read_history( WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::JanitorEvents(read) | WarehouseHistoryRead::Migrations(read) - | WarehouseHistoryRead::OldNameViews(read) => &read.location, + | WarehouseHistoryRead::OldNameViews(read) + | WarehouseHistoryRead::Microbatches(read) => &read.location, }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -114,6 +115,10 @@ fn read_page( .import("sqlbuild.executor.janitor_events.constants")? .getattr("JANITOR_EVENT_COLUMNS")? .extract()?, + WarehouseHistoryRead::Microbatches(_) => py + .import("sqlbuild.microbatches.constants")? + .getattr("MICROBATCH_COLUMNS")? + .extract()?, _ => Vec::new(), }; let event_columns: Vec<&str> = event_columns.iter().map(String::as_str).collect(); @@ -172,6 +177,15 @@ fn read_page( OLD_NAME_VIEW_COLUMNS.as_slice(), "created_at", ), + WarehouseHistoryRead::Microbatches(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_microbatches", + event_columns.as_slice(), + "created_at", + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -242,9 +256,9 @@ fn read_page( let key_columns: Option<&[&str]> = match request { WarehouseHistoryRead::AuditResults(_) => Some(&["result_id", "occurred_at"]), WarehouseHistoryRead::JanitorEvents(_) => Some(&["event_id", "occurred_at"]), - WarehouseHistoryRead::Migrations(_) | WarehouseHistoryRead::OldNameViews(_) => { - Some(&["event_id", "created_at"]) - } + WarehouseHistoryRead::Migrations(_) + | WarehouseHistoryRead::OldNameViews(_) + | WarehouseHistoryRead::Microbatches(_) => Some(&["event_id", "created_at"]), _ => None, }; let mut cursor_indexes: Vec = columns @@ -359,6 +373,7 @@ fn read_page( WarehouseHistoryRecord::JanitorEvent(record) => record.occurred_at, WarehouseHistoryRecord::Migration(record) => record.created_at, WarehouseHistoryRecord::OldNameView(record) => record.created_at, + WarehouseHistoryRecord::Microbatch(record) => record.created_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -405,6 +420,22 @@ fn decode_record( WarehouseHistoryRead::OldNameViews(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp, "expires_at"])); } + WarehouseHistoryRead::Microbatches(read) => { + return decode_event_record( + &read.location, + row, + ( + columns, + &[ + timestamp, + "origin_run_started_at", + "execution_run_started_at", + "completed_at", + "observed_at", + ], + ), + ); + } }; let mut value = json!({ "state_database": location.database, diff --git a/fensu.toml b/fensu.toml index 8181b019b1..c614ff7a70 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1662,7 +1662,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index ff26bb9750..4214a28885 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -46,6 +46,10 @@ ) from sqlbuild.executor.janitor_events.models import JanitorEventRecord from sqlbuild.executor.janitor_events.types import JanitorEventType +from sqlbuild.microbatches.classes.direct_store import DirectMicrobatchEventStore +from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec +from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchWriteResult from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -190,6 +194,27 @@ def append_janitor_history(adapter: Any, connection: Any, fixture: dict[str, Any writer.write(connection=connection, records=(records[0],) * fixture["duplicate_rows"]) +def append_microbatch_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real append-only coordination writer; reading never enters the executor's state path.""" + rows: list[dict[str, Any]] = [ + fixture["record"] + | {"event_id": f"{fixture['id_prefix']}-{index:03}"} + | fixture["record_overrides"].get(f"{fixture['id_prefix']}-{index:03}", {}) + for index in range(fixture["source_rows"]) + ] + records: list[MicrobatchEvent] = [] + for row in rows: + values: tuple[Any, ...] = tuple(row[column] for column in MICROBATCH_COLUMNS) + records.append(MicrobatchEventCodec.from_row(values)) + writer: DirectMicrobatchEventStore = DirectMicrobatchEventStore( + adapter=adapter, connection=connection + ) + first: MicrobatchWriteResult = writer.write_many(tuple(records)) + assert first.inserted == fixture["source_rows"] + repeated: MicrobatchWriteResult = writer.write_many((records[0],) * fixture["duplicate_rows"]) + assert repeated.inserted == 0 + + def append_migration_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: """Real model-move facts, including idempotent duplicate attempts.""" data: dict[str, Any] = migration_history_data(fixture, ("origin", "destination")) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index ab211ffd66..8e5ce46011 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -47,6 +47,7 @@ append_contended_fingerprints, append_janitor_history, append_legacy_history, + append_microbatch_history, append_migration_history, append_old_name_history, append_retried_fingerprints, @@ -100,6 +101,8 @@ ("audit_description", "audit_meta_json"), ), ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + # SQLite has no supported concurrent-microbatch coordination writer. + ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), ( "duckdb", "database", @@ -171,7 +174,7 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and } expected: dict[str, Any] = fixture["record"] | dict.fromkeys(legacy_columns) assert rows[0] == expected - assert rows[0]["run_id"] == test_case.expected_run_id + assert rows[0][fixture.get("run_key", "run_id")] == test_case.expected_run_id assert rows[fixture["later_index"]] == ( expected | fixture["later_record"] | dict.fromkeys(legacy_columns) ) @@ -236,6 +239,7 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and ("audit_description", "audit_meta_json"), ), ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), ( "duckdb", "database", @@ -333,7 +337,7 @@ def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retaine | {source_key: record[source_key]} | fixture["record_overrides"].get(record[source_key], {}) ) - assert record["run_id"] == test_case.expected_run_id + assert record[fixture.get("run_key", "run_id")] == test_case.expected_run_id assert identities.setdefault(record[source_key], event_id) == event_id with closing(sqlite3.connect(control)) as database: _ = database.execute(mirror_sql) From 389adf1baafd310fffdaef8d17125f443daaafd0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:43:12 +0000 Subject: [PATCH 094/138] feat: trigger sensor runs from new source data --- .../orchestration/source_data_sensor.json | 26 ++ .../edge_cases/orchestration/source_orders.py | 17 + .../orchestration/source_orders.yml | 8 + clients/typescript/src/schema.d.ts | 4 + clients/typescript/tests/types.ts | 7 + .../src/sensors/_helpers/conditions.rs | 5 +- .../src/sensors/_helpers/nodes.rs | 58 ++- .../src/sensors/constants.rs | 1 + .../src/sensors/main/validate_sensors.rs | 12 +- .../src/sensors/models.rs | 12 + .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 11 +- .../src/store/source_sensors.sql | 3 + .../warehouse_metadata/_helpers/freshness.rs | 71 +++- .../main/bootstrap_source_observations.rs | 54 +++ .../main/ingest_source_freshness.rs | 8 +- .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 18 + .../tests/credentials/test_audit.rs | 2 + .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_sources.rs | 379 ++++++++++++++++++ .../tests/store/helpers.rs | 2 + .../_helpers/native_host/constants.rs | 1 + .../native_host/orchestration_declarations.rs | 1 + .../native_host/orchestration_triggers.rs | 16 +- src/sqlbuild/_native.pyi | 2 +- src/sqlbuild/orchestration.py | 6 + .../models/authoring.py | 18 +- .../publication/test_source_sensor.py | 161 ++++++++ .../test_orchestration_snapshot.py | 45 +++ 30 files changed, 934 insertions(+), 18 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/source_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/source_orders.yml create mode 100644 crates/sqlbuild-orchestration/src/store/source_sensors.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/bootstrap_source_observations.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_sources.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json b/benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json new file mode 100644 index 0000000000..6cbb034c46 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json @@ -0,0 +1,26 @@ +{ + "record": { + "event_id": "orders-source-000", + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders-🧾", + "strategy": "max_column", + "value_kind": "integer", + "data_version": "18446744073709551617", + "data_version_hash": "orders-version-1", + "observed_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "null_indices": [1, 102], + "unselected_index": 103, + "late_record": { + "event_id": "orders-source-late", + "data_version": "18446744073709551619", + "data_version_hash": "orders-version-2", + "observed_at": "2026-01-01T01:02:03Z" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/source_orders.py new file mode 100644 index 0000000000..1b86cc72b9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders.py @@ -0,0 +1,17 @@ +"""Consume changed warehouse source versions without treating a sync poll as new data.""" + +from sqlbuild.orchestration import job, sensor, source_new_data + +orders_refresh = job( + "orders_refresh", + description="Refresh orders for a changed source version.", + select="orders", + overlap="allow", +) +on_source_orders = sensor( + "on_source_orders", + description="Refresh orders for the first source version and each changed version.", + when=source_new_data(select="source:raw_orders"), + run=orders_refresh, + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders.yml b/benchmarks/duckshop/edge_cases/orchestration/source_orders.yml new file mode 100644 index 0000000000..17fc10a3dd --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders.yml @@ -0,0 +1,8 @@ +sources: + - name: raw_orders + description: Raw orders. + schema: main + table: raw_orders + columns: + - name: order_id + type: BIGINT diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index df3974d453..de4d9896a0 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1270,6 +1270,10 @@ export interface components { kind: "node_event"; kinds: components["schemas"]["NodeEventKind"][]; selection: string[]; + } | { + /** @enum {string} */ + kind: "source_new_data"; + selection: string[]; } | { code_hash: string; function: string; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 87991ccd5c..914535061d 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -67,6 +67,13 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/microba params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const sourceCondition: components["schemas"]["SensorCondition"] = { + kind: "source_new_data", selection: ["source:raw_orders"], +}; +void sourceCondition; +// @ts-expect-error Source conditions select sources, not execution event kinds. +const invalidSourceCondition: components["schemas"]["SensorCondition"] = { kind: "source_new_data", selection: ["source:raw_orders"], kinds: ["success"] }; +void invalidSourceCondition; // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs index b5b15de45e..5f46f2912d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::sensors::_helpers::nodes::consume_node_events; +use crate::sensors::_helpers::nodes::{consume_node_events, consume_source_events}; use crate::sensors::_helpers::platform::consume_platform_events; use crate::sensors::_helpers::runs::consume_run_events; use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; @@ -23,6 +23,9 @@ pub(crate) async fn consume_condition( SensorCondition::NodeEvent { selection, kinds } => { consume_node_events(transaction, evaluation, selection, kinds).await } + SensorCondition::SourceNewData { selection } => { + consume_source_events(transaction, evaluation, selection).await + } SensorCondition::RunEvent { job_key, kinds, diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs index 0f2b89f63a..32ae854d69 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -1,4 +1,4 @@ -//! Node transitions require earlier immutable start evidence from the same run claim. +//! Node conditions consume immutable execution evidence or warehouse source observations. use std::collections::BTreeSet; @@ -8,7 +8,7 @@ use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::_helpers::transitions::matches_transition; use crate::sensors::models::{ - NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, + NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, SourceSensorEvent, }; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -75,6 +75,60 @@ pub(crate) async fn consume_node_events( Ok((transaction, (cursor, run_ids))) } +pub(crate) async fn consume_source_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT storage_id,event_id,invocation_id,invocation_sequence,payload FROM recall_events \ + WHERE producer='warehouse' AND event_type='warehouse_source_freshness' \ + AND invocation_id LIKE 'warehouse/source-freshness/%' AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", + vec![Parameter::Integer(evaluation.cursor.event_storage_id)], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let event: SourceSensorEvent = decode_row(row)?; + cursor.event_storage_id = event.storage_id; + let observation = &event.payload; + let record = &observation.record; + if observation.location_id != evaluation.location_id + || observation.target != evaluation.target + || !selection.contains(&format!("source:{}", record.source_name)) + || record.data_version.is_none() + { + continue; + } + let previous; + (transaction, previous) = query( + transaction, + "SELECT payload -> 'record' ->> 'data_version_hash' AS hash FROM recall_events \ + WHERE producer='warehouse' AND event_type='warehouse_source_freshness' \ + AND invocation_id=$1 AND invocation_sequence=$2 LIMIT 1", + vec![ + Parameter::Text(event.invocation_id), + Parameter::Integer(event.invocation_sequence - 1), + ], + ) + .await?; + if previous + .first() + .is_some_and(|row| row["hash"] == record.data_version_hash) + { + continue; + } + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + Ok((transaction, (cursor, run_ids))) +} + async fn prior_start( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs index c74670d166..87d76698ad 100644 --- a/crates/sqlbuild-orchestration/src/sensors/constants.rs +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -2,4 +2,5 @@ pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; pub(crate) const CUSTOM_SENSOR_RECHECK_SECONDS: i64 = 1; +pub(crate) const SOURCE_NODE_KIND: &str = "source"; pub const MAX_SENSOR_RUN_REQUESTS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index da488cc75d..8224fc177c 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -1,6 +1,7 @@ //! Native publication validates every sensor before any snapshot pointer moves. use crate::declarations::models::Definitions; +use crate::sensors::constants::SOURCE_NODE_KIND; use crate::sensors::models::SensorCondition; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; @@ -21,6 +22,15 @@ pub(crate) fn validate_sensors( && !selection.is_empty() && selection.iter().all(|key| input.nodes.contains_key(key)) } + SensorCondition::SourceNewData { selection } => { + !selection.is_empty() + && selection.iter().all(|key| { + input + .nodes + .get(key) + .is_some_and(|node| node.kind == SOURCE_NODE_KIND) + }) + } SensorCondition::RunEvent { job_key, kinds, @@ -45,7 +55,7 @@ pub(crate) fn validate_sensors( && !definitions.groups.contains_key(&sensor.job_key)) { return Err(ControlError::Invalid(format!( - "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes" + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes; source_new_data must select nonempty snapshot sources" ))); } } diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 9c6a42d596..b53adb7a29 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -45,6 +45,9 @@ pub enum SensorCondition { selection: BTreeSet, kinds: BTreeSet, }, + SourceNewData { + selection: BTreeSet, + }, Custom { function: String, code_hash: String, @@ -106,6 +109,15 @@ pub(crate) struct NodeSensorEvent { pub(crate) payload: serde_json::Value, } +#[derive(Debug, Deserialize)] +pub(crate) struct SourceSensorEvent { + pub(crate) event_id: Uuid, + pub(crate) invocation_id: String, + pub(crate) invocation_sequence: i64, + pub(crate) storage_id: i64, + pub(crate) payload: crate::warehouse_metadata::models::SourceFreshnessObservation, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct SensorDefinition { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index a6c5364fbc..1ff60fe91c 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -31,6 +31,7 @@ pub(crate) const WAREHOUSE_AUDIT_RESULTS: &str = include_str!("../warehouse_audi pub(crate) const WAREHOUSE_JANITOR_EVENTS: &str = include_str!("../warehouse_janitor_events.sql"); pub(crate) const WAREHOUSE_MIGRATIONS: &str = include_str!("../warehouse_migrations.sql"); pub(crate) const WAREHOUSE_MICROBATCHES: &str = include_str!("../warehouse_microbatches.sql"); +pub(crate) const SOURCE_SENSORS: &str = include_str!("../source_sensors.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index af405ba7c5..54bc4c78da 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -7,9 +7,9 @@ use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, - WAREHOUSE_RECONCILIATION, render, verify_migrations, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, SOURCE_SENSORS, WAREHOUSE_AUDIT_RESULTS, + WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, + WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -17,6 +17,7 @@ use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::main::bootstrap_source_observations::bootstrap_source_observations; /// Create the schema once, or reject drift and schemas newer than this server supports. pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { @@ -56,6 +57,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_janitor_events", WAREHOUSE_JANITOR_EVENTS), ("warehouse_migrations", WAREHOUSE_MIGRATIONS), ("warehouse_microbatches", WAREHOUSE_MICROBATCHES), + ("source_sensors", SOURCE_SENSORS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); @@ -91,6 +93,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { { (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; } + if sql == SOURCE_SENSORS { + transaction = bootstrap_source_observations(transaction).await?; + } let now = transaction.now; (transaction, _) = query(transaction, "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ diff --git a/crates/sqlbuild-orchestration/src/store/source_sensors.sql b/crates/sqlbuild-orchestration/src/store/source_sensors.sql new file mode 100644 index 0000000000..f1b02088fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/source_sensors.sql @@ -0,0 +1,3 @@ +CREATE INDEX recall_warehouse_sources ON recall_events (storage_id) + WHERE producer='warehouse' AND event_type='warehouse_source_freshness' + AND invocation_id LIKE 'warehouse/source-freshness/%'; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs index 9967fd9183..1913e78ba6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs @@ -1,8 +1,17 @@ -//! Validate bounded freshness observations and their optional reconciliation scope. +//! Validate source observations and sequence their receipt for durable sensor consumption. + +use chrono::SubsecRound; +use serde_json::json; +use uuid::Uuid; use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; use crate::warehouse_metadata::_helpers::ingestion::validate_fields; -use crate::warehouse_metadata::models::SourceFreshnessBatch; +use crate::warehouse_metadata::models::{ + SourceFreshnessBatch, SourceFreshnessObservation, WarehouseFreshnessRecord, +}; pub(crate) fn validate_batch(batch: &SourceFreshnessBatch) -> Result<(), ControlError> { let invalid_identity = batch @@ -39,3 +48,61 @@ pub(crate) fn validate_batch(batch: &SourceFreshnessBatch) -> Result<(), Control invalid_scope, ) } + +pub(crate) async fn append_observation( + transaction: ControlTransaction, + (location_id, target, source_id): (Uuid, &str, Uuid), + record: &WarehouseFreshnessRecord, +) -> Result { + if record.data_version.is_none() { + return Ok(transaction); + } + let event_id = Uuid::new_v5(&source_id, b"source-freshness-observation"); + let occurred_at = record.observed_at.trunc_subsecs(6); + let (transaction, existing) = query( + transaction, + "SELECT event_id FROM recall_events WHERE occurred_at=$1 AND event_id=$2 LIMIT 1", + vec![ + Parameter::Time(occurred_at), + Parameter::Text(event_id.to_string()), + ], + ) + .await?; + if !existing.is_empty() { + return Ok(transaction); + } + let scope = serde_json::to_vec(&( + target, + &record.state_database, + &record.state_schema, + &record.source_name, + &record.target_database, + &record.target_schema, + &record.target_name, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let source = Uuid::new_v5(&location_id, &scope); + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("warehouse/source-freshness/{source}"), + event_type: "warehouse_source_freshness", + producer: "warehouse", + occurred_at, + run_id: None, + payload: json!(SourceFreshnessObservation { + location_id, + target: target.into(), + record: record.clone(), + }), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "source observation conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/bootstrap_source_observations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/bootstrap_source_observations.rs new file mode 100644 index 0000000000..6d6078c0b8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/bootstrap_source_observations.rs @@ -0,0 +1,54 @@ +//! Sequence preexisting observations inside migration's writer fence, before new imports. + +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::freshness::append_observation; +use crate::warehouse_metadata::models::StoredSourceFreshnessObservation; + +pub(crate) async fn bootstrap_source_observations( + mut transaction: ControlTransaction, +) -> Result { + let mut after: Vec = Vec::new(); + loop { + let predicate = if after.is_empty() { + "" + } else { + "WHERE (location_id,target,observed_at,event_id)>($1,$2,$3,$4)" + }; + let rows; + (transaction, rows) = query( + transaction, + &format!( + "SELECT event_id,location_id,target,observed_at,record_json \ + FROM sqb_warehouse_freshness {predicate} \ + ORDER BY location_id,target,observed_at,event_id LIMIT 100" + ), + after, + ) + .await?; + if rows.is_empty() { + return Ok(transaction); + } + after = Vec::new(); + for row in rows { + let observation: StoredSourceFreshnessObservation = decode_row(row)?; + after = vec![ + Parameter::Id(observation.location_id), + Parameter::Text(observation.target.clone()), + Parameter::Time(observation.observed_at), + Parameter::Id(observation.event_id), + ]; + transaction = append_observation( + transaction, + ( + observation.location_id, + &observation.target, + observation.event_id, + ), + &observation.record_json, + ) + .await?; + } + } +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index de7e023696..173f08a11a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -8,7 +8,7 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter}; -use crate::warehouse_metadata::_helpers::freshness::validate_batch; +use crate::warehouse_metadata::_helpers::freshness::{append_observation, validate_batch}; use crate::warehouse_metadata::_helpers::ingestion::{ begin_ingestion, commit_ingestion, scoped_event_id, }; @@ -75,7 +75,11 @@ pub async fn ingest_source_freshness( vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; - applied |= !inserted.is_empty(); + if !inserted.is_empty() { + transaction = + append_observation(transaction, (location_id, target, event_id), record).await?; + applied = true; + } } commit_ingestion( transaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index b249e5a7f0..1eefd96bc0 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod bootstrap_source_observations; pub mod ingest_audit_results; pub mod ingest_fingerprints; pub mod ingest_janitor_events; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index f216ca085a..cc04480bf3 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -23,6 +23,24 @@ pub struct WarehouseFreshnessRecord { pub observed_at: DateTime, } +/// Receipt order is sensor progress; warehouse observation time remains source evidence. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SourceFreshnessObservation { + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) record: WarehouseFreshnessRecord, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredSourceFreshnessObservation { + pub(crate) event_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) observed_at: DateTime, + pub(crate) record_json: WarehouseFreshnessRecord, +} + /// A bounded page from one registered reader of a published location/target generation. #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 84d0eea7ab..61472d4374 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -124,6 +124,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP INDEX recall_warehouse_sources", + "DELETE FROM sqb_schema_migrations WHERE name='source_sensors'", "DROP TABLE sqb_warehouse_microbatches", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_microbatches'", "DROP TABLE sqb_warehouse_old_name_views", diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 98c7d70d14..e97b015a8c 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -14,6 +14,8 @@ mod test_nodes; mod test_recovery; #[path = "test_runs.rs"] mod test_runs; +#[path = "test_sources.rs"] +mod test_sources; #[path = "test_transitions.rs"] mod test_transitions; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs new file mode 100644 index 0000000000..7a86cc8e4d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -0,0 +1,379 @@ +//! Warehouse imports, atomic requests and durable receipt cursors run against both real stores. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::snapshot; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::models::{SourceFreshnessBatch, WarehouseFreshnessRecord}; + +#[test] +fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_rebuilds_do_not_repeat() + { + let test_cases = [ + SensorTestCase { + description: "SQLite source versions", + backend: Backend::Sqlite, + expected_runs: 2, + }, + SensorTestCase { + description: "PostgreSQL source versions", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.insert( + "source:raw_orders".into(), + SnapshotNode { + kind: "source".into(), + definition: json!({"name": "raw_orders"}), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ); + input.documents.insert( + "triggers".into(), + json!({"orders_changed": { + "description": "Refresh orders after source data changes.", "job_key": "orders", + "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, + "minimum_interval_seconds": 1 + }}), + ); + publish_snapshot(store, &input) + .await + .expect("publish source sensor"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json"); + let data: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(fixture_path).expect("DuckShop source sensor"), + ) + .expect("synthetic source versions"); + let mut records: Vec = Vec::new(); + for index in 0..data["source_rows"].as_u64().expect("row count") { + let mut record = data["record"].clone(); + record["event_id"] = json!(format!("orders-source-{index:03}")); + records.push( + serde_json::from_value::(record).expect("record"), + ); + } + for index in data["null_indices"].as_array().expect("null observations") { + let record = &mut records[index.as_u64().expect("index") as usize]; + record.data_version = None; + record.data_version_hash = "orders-null".into(); + } + records[data["unselected_index"].as_u64().expect("index") as usize].source_name = + "raw_products".into(); + let receipt_fault = TransactionFault { + description: "source receipt rollback", + table: "recall_events", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected source receipt failure')", + postgres_body: "RAISE EXCEPTION 'injected source receipt failure'", + }; + install_fault(store, &receipt_fault).await; + assert!( + ingest_source_freshness( + store, + (location, "development"), + &SourceFreshnessBatch { + session_id, + generation: 1, + records: records[..1].to_vec(), + reconciliation: None, + } + ) + .await + .is_err() + ); + remove_fault(store, &receipt_fault).await; + let (transaction, mirrored) = query( + begin(store, TransactionMode::Read) + .await + .expect("mirror read"), + "SELECT COUNT(*) AS count FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("rolled back mirror"); + commit(transaction).await.expect("finish read"); + assert_eq!(mirrored[0]["count"], 0); + for page in records.chunks(100) { + let batch = SourceFreshnessBatch { + session_id, + generation: 1, + records: page.to_vec(), + reconciliation: None, + }; + assert!( + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("import") + .applied + ); + assert!( + !ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("exact retry") + .applied + ); + } + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("pre-sensor predecessor"); + for sql in [ + "DELETE FROM recall_events WHERE producer='warehouse' AND event_type='warehouse_source_freshness'", + "DROP INDEX recall_warehouse_sources", + "DELETE FROM sqb_schema_migrations WHERE name='source_sensors'", + ] { + (transaction, _) = query(transaction, sql, vec![]) + .await + .expect("preserve mirror without source sensor receipts"); + } + commit(transaction).await.expect("predecessor committed"); + install_fault(store, &receipt_fault).await; + assert!(migrate_store(store).await.is_err()); + remove_fault(store, &receipt_fault).await; + migrate_store(store) + .await + .expect("bootstrap preexisting source observations"); + migrate_store(store) + .await + .expect("warm bootstrap is idempotent"); + for page in records.chunks(100) { + assert!( + !ingest_source_freshness( + store, + (location, "development"), + &SourceFreshnessBatch { + session_id, + generation: 1, + records: page.to_vec(), + reconciliation: None, + } + ) + .await + .expect("already mirrored observation retry") + .applied + ); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluation") + .expect("due"); + let fault = TransactionFault { + description: "source cursor rollback", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected source cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected source cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &pending) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback") + .is_empty() + ); + assert_eq!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("retry"), + Some(pending.clone()) + ); + assert_eq!( + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &pending) + ) + .await + .expect("bounded source page") + .expect("complete"), + 1 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &pending) + .await + .expect("lost acknowledgement retry"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("second page") + .expect("due"); + assert!(second.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &second) + .await + .expect("unchanged source versions"), + 0 + ); + let mut late = data["record"].as_object().expect("record").clone(); + late.extend( + data["late_record"] + .as_object() + .expect("late source observation") + .clone(), + ); + let batch = SourceFreshnessBatch { + session_id, + generation: 1, + records: vec![serde_json::from_value(json!(late)).expect("late record")], + reconciliation: None, + }; + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("late commit"); + tokio::time::sleep(Duration::from_secs(1)).await; + let later = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("late evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &later) + .await + .expect("late warehouse time must not skip new receipt"), + 1 + ); + let events = read_recall_events(store, 0, 1000) + .await + .expect("durable observations"); + let source_events = events + .iter() + .filter(|event| event["event_type"] == "warehouse_source_freshness") + .count(); + assert_eq!( + source_events, + records.len() + 1 + - data["null_indices"] + .as_array() + .expect("null observations") + .len() + ); + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("disposable mirror"), + "DELETE FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("clear mirror"); + commit(transaction).await.expect("clear mirror committed"); + for page in records.chunks(100) { + ingest_source_freshness( + store, + (location, "development"), + &SourceFreshnessBatch { + session_id, + generation: 1, + records: page.to_vec(), + reconciliation: None, + }, + ) + .await + .expect("rebuild mirror"); + } + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("rebuild late fact"); + tokio::time::sleep(Duration::from_secs(1)).await; + let rebuilt = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("rebuilt evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &rebuilt) + .await + .expect("rebuild has no new source facts"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("queued requests") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + let rebuilt = read_recall_events(store, 0, 1000) + .await + .expect("rebuilt observations"); + assert_eq!( + rebuilt + .iter() + .filter(|event| event["event_type"] == "warehouse_source_freshness") + .count(), + source_events + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 2246ec93e8..1ece736b2c 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP INDEX recall_warehouse_sources", + "DELETE FROM sqb_schema_migrations WHERE name='source_sensors'", "DROP TABLE sqb_warehouse_microbatches", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_microbatches'", "DROP TABLE sqb_warehouse_old_name_views", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 035588e5e9..74e15e69db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,6 +4,7 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; +pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; /// The sensor context is supplied by the host, not a provider. pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 12796bb6fe..7b3acd2849 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -50,6 +50,7 @@ pub(crate) struct AuthoredGroup { #[derive(FromPyObject)] pub(crate) struct AuthoredSensorCondition { pub(crate) kind: String, + #[pyo3(default)] pub(crate) kinds: Vec, #[pyo3(default)] pub(crate) job: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 256be90f0c..519768dd66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -21,7 +21,9 @@ use sqlbuild_orchestration::sensors::models::SensorDefinition; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; -use crate::bindings::_helpers::native_host::constants::CALENDAR_WINDOW_PARAMETER; +use crate::bindings::_helpers::native_host::constants::{ + CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, +}; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredSensor, authored_setting, @@ -276,14 +278,17 @@ pub(crate) fn compile_sensors( } let condition = match (sensor.when, sensor.function) { (Some(when), None) => { - let mut condition = json!({"kind": when.kind, "kinds": when.kinds}); + let mut condition = json!({"kind": when.kind}); + if when.kind != SOURCE_NEW_DATA_CONDITION { + condition["kinds"] = json!(when.kinds); + } if let Some(job) = when.job { condition["job_key"] = json!(job.name); } if let Some(select) = when.select { let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; } if let Some(selectors) = when.executed_successfully { condition["executed_successfully"] = @@ -328,5 +333,10 @@ fn sensor_nodes(graph: &ProjectGraph, selectors: &[String], field: &str) -> PyRe "{field} needs nonempty selectors" ))); } + if field == SOURCE_NEW_DATA_CONDITION && nodes.iter().any(|key| !key.starts_with("source:")) { + return Err(PyValueError::new_err( + "source_new_data must select only source nodes", + )); + } Ok(json!(nodes)) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 52f5fbd788..2ee216eef3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1740,7 +1740,7 @@ class NativeCompiledAudit: def meta(self) -> dict[str, object] | None: ... class NativeOrchestrationSnapshot: - """Compiled jobs, schedules and sensors retained as data until native publication.""" + """Compiled jobs, schedules and event/source-version sensors retained as publishable data.""" def to_json(self) -> str: ... def publish(self, server_url: str, bearer: str) -> str: ... diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 568ce36709..c176f6f09a 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -13,6 +13,7 @@ Schedule, Sensor, SkipReason, + SourceNewDataCondition, ) from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule as calendar_schedule, @@ -26,6 +27,9 @@ from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor +from sqlbuild.orchestration_declarations.models.authoring import ( + SourceNewDataCondition as source_new_data, +) from sqlbuild.orchestration_declarations.models.sensor_context import SensorContext from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind @@ -45,6 +49,7 @@ "Sensor", "SensorContext", "SkipReason", + "SourceNewDataCondition", "calendar_schedule", "compile_snapshot", "job", @@ -54,4 +59,5 @@ "run_event", "schedule", "sensor", + "source_new_data", ) diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index fd188208ed..fc5fcca6cb 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -109,15 +109,27 @@ class NodeEventCondition: kind: Literal["node_event"] = field(default="node_event", init=False) +@dataclass(frozen=True, kw_only=True) +class SourceNewDataCondition: + """Match the first non-null source version and later changed versions, in receipt order.""" + + select: str + kind: Literal["source_new_data"] = field(default="source_new_data", init=False) + + @dataclass(frozen=True, eq=False) class Sensor: """Start a job for each matching event; event identity deduplicates replayed requests.""" name: str description: str = field(kw_only=True) - when: PlatformEventCondition | RunEventCondition | NodeEventCondition | None = field( - default=None, kw_only=True - ) + when: ( + PlatformEventCondition + | RunEventCondition + | NodeEventCondition + | SourceNewDataCondition + | None + ) = field(default=None, kw_only=True) run: Job | JobGroup = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py new file mode 100644 index 0000000000..7bf8f09885 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -0,0 +1,161 @@ +"""An external warehouse writer drives a real sensor, queue and isolated DuckDB worker.""" + +import json +import os +import sqlite3 +import sys +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from sqlbuild._native import ( + SourceFreshnessRecord, + SourceFreshnessRenderers, + write_source_freshness_records, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + scheduled_worker, + wait_for_generation, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "changed external source versions", + "SELECT 73 AS amount", + "SELECT amount FROM orders", + "success", + [(73,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_external_source_versions_when_a_location_resyncs_then_only_changed_data_starts_real_work( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json").read_text() + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + for phase, (source, count) in enumerate( + ( + (data["record"], data["source_rows"]), + (data["record"] | data["late_record"], 1), + ) + ): + values: dict[str, Any] = source.copy() + for key in ("event_id", "state_database", "state_schema", "observed_at"): + values.pop(key) + records: tuple[SourceFreshnessRecord, ...] = tuple( + SourceFreshnessRecord( + **( + values + | { + "run_id": f"{source['run_id']}-{phase}-{index}", + "observed_at": datetime.fromisoformat(source["observed_at"]) + + timedelta(microseconds=index), + } + ) + ) + for index in range(count) + ) + adapter: Any = DuckDbAdapter() + connection: Any = adapter.connect( + {"database": str(worker.project / "warehouse.duckdb")} + ) + renderers: SourceFreshnessRenderers = SourceFreshnessRenderers( + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + try: + for batch in (records, records[:1] * 3): + write_source_freshness_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=batch, + renderers=renderers, + ) + finally: + adapter.close(connection) + log: Path = tmp_path / f"location-{phase}.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / f"location-{phase}.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": worker.api.bearer}, + ) as location: + _ = wait_for_generation(worker.api, "development", phase + 2, location, log) + deadline: float = time.monotonic() + 30 + observed: int = 0 + runs: list[tuple[str]] = [] + while observed != data["source_rows"] + phase or len(runs) != phase + 1: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + observed = control.execute( + "SELECT COUNT(*) FROM sqb_warehouse_freshness" + ).fetchone()[0] + runs = control.execute( + "SELECT run_id FROM sqb_runs WHERE origin='sensor' ORDER BY queued_at,run_id" + ).fetchall() + assert len(runs) <= phase + 1, ( + "unchanged observations and exact retries must not create runs" + ) + time.sleep(0.05) + result: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=runs[-1][0] + ) + assert result["status"] == test_case.expected_status + assert result["job_key"] == "orders_refresh" + assert result["node_counts_json"] == {"success": 1} + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert ( + warehouse.execute(test_case.warehouse_query).fetchall() + == test_case.expected_rows + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT COUNT(*) FROM sqb_run_requests").fetchone() == (2,) + assert control.execute( + "SELECT COUNT(*) FROM recall_events WHERE producer='warehouse'" + ).fetchone() == (105,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index bd973f07a7..64638b3774 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -741,6 +741,27 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="source_new_data selectors freeze only source nodes without event kinds", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_jobs=("orders_refresh",), + expected_sensors={ + "on_source_orders": { + "description": "Refresh orders for the first source version and each changed version.", + "job_key": "orders_refresh", + "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, + "minimum_interval_seconds": 1, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", files={ @@ -990,6 +1011,30 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="unknown variant", ), + SnapshotDiscoveryRejectionTestCase( + description="source_new_data rejects model selections", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ) + .read_text() + .replace('select="source:raw_orders"', 'select="orders"'), + }, + expected_error=ValueError, + expected_message="source_new_data must select only source nodes", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_new_data rejects blank selections", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ) + .read_text() + .replace('select="source:raw_orders"', 'select=" "'), + }, + expected_error=ValueError, + expected_message="source_new_data needs nonempty selectors", + ), SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", files={ From bdd41a1d00a34167011524032b6d15891bc85d03 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 13:55:43 +0000 Subject: [PATCH 095/138] feat: trigger source sensors from timestamp age --- .../orchestration/managed_orders.yml | 9 + .../edge_cases/orchestration/orders_loader.py | 7 + .../orchestration/source_stale_journey.json | 24 ++ .../orchestration/source_stale_sensor.json | 25 ++ .../edge_cases/orchestration/stale_orders.py | 13 + clients/typescript/src/schema.d.ts | 6 + clients/typescript/tests/types.ts | 7 + .../src/sensors/_helpers/conditions.rs | 8 +- .../src/sensors/_helpers/nodes.rs | 114 +++++++++ .../src/sensors/constants.rs | 1 + .../src/sensors/main/validate_sensors.rs | 5 +- .../src/sensors/models.rs | 5 + .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 8 +- .../src/store/source_staleness.sql | 5 + .../tests/credentials/test_audit.rs | 2 + .../tests/sensors/test_sources.rs | 240 ++++++++++++++++++ .../tests/store/helpers.rs | 2 + .../_helpers/native_host/constants.rs | 1 + .../native_host/orchestration_declarations.rs | 2 + .../native_host/orchestration_triggers.rs | 16 +- src/sqlbuild/_native.pyi | 2 +- src/sqlbuild/orchestration.py | 6 + .../models/authoring.py | 10 + .../publication/test_source_sensor.py | 31 ++- .../test_orchestration_snapshot.py | 82 ++++++ 26 files changed, 617 insertions(+), 15 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/orders_loader.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/stale_orders.py create mode 100644 crates/sqlbuild-orchestration/src/store/source_staleness.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml b/benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml new file mode 100644 index 0000000000..4ea1d73bb7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml @@ -0,0 +1,9 @@ +sources: + - name: raw_orders + description: Orders inheriting the managed loader namespace. + managed: true + write_strategy: table + table: raw_orders + columns: + - name: order_id + type: BIGINT diff --git a/benchmarks/duckshop/edge_cases/orchestration/orders_loader.py b/benchmarks/duckshop/edge_cases/orchestration/orders_loader.py new file mode 100644 index 0000000000..ad4c8b828d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/orders_loader.py @@ -0,0 +1,7 @@ +from sqlbuild.loaders import loader + + +@loader +def raw_orders(ctx): + """Load synthetic orders in the target loader namespace.""" + return [{"order_id": 1}] diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json b/benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json new file mode 100644 index 0000000000..e993214129 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json @@ -0,0 +1,24 @@ +{ + "record": { + "event_id": "orders-source-000", + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders", + "strategy": "max_column", + "value_kind": "timestamp", + "data_version": "2026-02-03T03:00:00+00:00", + "data_version_hash": "orders-timestamp-1", + "observed_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "late_record": { + "event_id": "orders-source-later", + "data_version": "2026-02-03T03:01:00+00:00", + "data_version_hash": "orders-timestamp-2", + "observed_at": "2026-02-03T04:05:07Z" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json b/benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json new file mode 100644 index 0000000000..25ae61920d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json @@ -0,0 +1,25 @@ +{ + "after_seconds": 60, + "source_count": 104, + "source_definitions": { + "source:raw_orders_103": { + "database": null, "schema": null, "table": "raw_orders", + "namespace": ["orders_data", "landing"] + } + }, + "cases": [ + {"index": 0, "age_seconds": 61, "offset_seconds": 10800}, + {"index": 1, "age_seconds": 60}, + {"index": 2, "age_seconds": 59}, + {"index": 3, "age_seconds": 600}, + {"index": 3, "age_seconds": 600, "null": true, "observed_offset_microseconds": 1}, + {"index": 4, "age_seconds": 600, "value_kind": "integer", "data_version": "18446744073709551617"}, + {"index": 5, "age_seconds": 600, "target_name": "archived_orders"}, + {"index": 103, "age_seconds": 61, "format": "%Y-%m-%dT%H:%M:%S%.f", "target_database": "orders_data", "target_schema": "landing"} + ], + "later_pages": [ + {"expected_cursor": "source:raw_orders_099", "expected_runs": 2}, + {"expected_cursor": null, "expected_runs": 4}, + {"expected_cursor": "source:raw_orders_099", "expected_runs": 4} + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py b/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py new file mode 100644 index 0000000000..4fc5b934a2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py @@ -0,0 +1,13 @@ +from sqlbuild.orchestration import Job, Sensor, source_stale + +orders_refresh = Job( + "orders_refresh", description="Refresh orders from stale timestamp data.", select="orders" +) + +on_stale_orders = Sensor( + "on_stale_orders", + description="Refresh orders once for each timestamp version older than one minute.", + when=source_stale(select="source:raw_orders", after_seconds=60), + run=orders_refresh, + minimum_interval_seconds=1, +) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index de4d9896a0..040acca1e8 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1274,6 +1274,12 @@ export interface components { /** @enum {string} */ kind: "source_new_data"; selection: string[]; + } | { + /** Format: int32 */ + after_seconds: number; + /** @enum {string} */ + kind: "source_stale"; + selection: string[]; } | { code_hash: string; function: string; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 914535061d..6b2df6e31d 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -74,6 +74,13 @@ void sourceCondition; // @ts-expect-error Source conditions select sources, not execution event kinds. const invalidSourceCondition: components["schemas"]["SensorCondition"] = { kind: "source_new_data", selection: ["source:raw_orders"], kinds: ["success"] }; void invalidSourceCondition; +const staleCondition: components["schemas"]["SensorCondition"] = { + kind: "source_stale", selection: ["source:raw_orders"], after_seconds: 60, +}; +void staleCondition; +// @ts-expect-error Staleness conditions require an explicit age threshold. +const missingStaleThreshold: components["schemas"]["SensorCondition"] = { kind: "source_stale", selection: ["source:raw_orders"] }; +void missingStaleThreshold; // @ts-expect-error Session mutations require the CSRF header. void client.POST("/api/auth/logout", {}); // @ts-expect-error Changing a password also requires the current password. diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs index 5f46f2912d..6b5ba9f245 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -2,7 +2,9 @@ use uuid::Uuid; -use crate::sensors::_helpers::nodes::{consume_node_events, consume_source_events}; +use crate::sensors::_helpers::nodes::{ + consume_node_events, consume_source_events, consume_stale_sources, +}; use crate::sensors::_helpers::platform::consume_platform_events; use crate::sensors::_helpers::runs::consume_run_events; use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; @@ -26,6 +28,10 @@ pub(crate) async fn consume_condition( SensorCondition::SourceNewData { selection } => { consume_source_events(transaction, evaluation, selection).await } + SensorCondition::SourceStale { + selection, + after_seconds, + } => consume_stale_sources(transaction, evaluation, selection, *after_seconds).await, SensorCondition::RunEvent { job_key, kinds, diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs index 32ae854d69..b556219f83 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -1,19 +1,25 @@ //! Node conditions consume immutable execution evidence or warehouse source observations. use std::collections::BTreeSet; +use std::num::NonZeroU32; +use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::_helpers::transitions::matches_transition; +use crate::sensors::constants::TIMESTAMP_VALUE_KIND; use crate::sensors::models::{ NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, SourceSensorEvent, }; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::models::WarehouseFreshnessRecord; pub(crate) async fn consume_node_events( transaction: ControlTransaction, @@ -129,6 +135,114 @@ pub(crate) async fn consume_source_events( Ok((transaction, (cursor, run_ids))) } +fn stale_version( + record: &WarehouseFreshnessRecord, + now: DateTime, + after_seconds: NonZeroU32, +) -> Result { + let Some(version) = record + .data_version + .as_deref() + .filter(|_| record.value_kind == TIMESTAMP_VALUE_KIND) + else { + return Ok(false); + }; + let instant = DateTime::parse_from_rfc3339(version) + .map(|value| value.with_timezone(&Utc)) + .or_else(|_| { + NaiveDateTime::parse_from_str(version, "%Y-%m-%dT%H:%M:%S%.f") + .map(|value| value.and_utc()) + }) + .map_err(|_| { + ControlError::Invalid( + "timestamp source freshness version is not an ISO datetime".into(), + ) + })?; + Ok(now.signed_duration_since(instant) > TimeDelta::seconds(i64::from(after_seconds.get()))) +} + +/// Reevaluate current timestamp age; only page position and idempotent requests are stored. +pub(crate) async fn consume_stale_sources( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, + after_seconds: NonZeroU32, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let mut cursor = evaluation.cursor.clone(); + let mut remaining = selection + .iter() + .filter(|key| cursor.custom.as_ref().is_none_or(|after| *key > after)); + let page: BTreeSet = remaining.by_ref().take(100).cloned().collect(); + let next = remaining + .next() + .is_some() + .then(|| page.last().cloned()) + .flatten(); + let (mut transaction, nodes) = + read_snapshot_nodes(transaction, evaluation.snapshot_id, Some(&page)).await?; + let mut run_ids: Vec = Vec::new(); + for (key, node) in nodes { + let mut parameters = vec![ + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + Parameter::Text(key.replacen("source:", "", 1)), + ]; + let mut predicate = String::new(); + for (column, value) in [ + ( + "target_database", + node.definition["namespace"] + .get(0) + .unwrap_or(&node.definition["database"]), + ), + ( + "target_schema", + node.definition["namespace"] + .get(1) + .unwrap_or(&node.definition["schema"]), + ), + ("target_name", &node.definition["table"]), + ] { + if let Some(value) = value.as_str() { + parameters.push(Parameter::Text(value.into())); + predicate.push_str(&format!( + " AND record_json ->> '{column}'=${}", + parameters.len() + )); + } else { + predicate.push_str(&format!(" AND record_json ->> '{column}' IS NULL")); + } + } + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT record_json FROM sqb_warehouse_freshness WHERE location_id=$1 AND target=$2 \ + AND record_json ->> 'source_name'=$3{predicate} ORDER BY observed_at DESC,event_id DESC LIMIT 1" + ), parameters).await?; + let Some(row) = rows.first() else { continue }; + let record: WarehouseFreshnessRecord = + serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if !stale_version(&record, evaluation.requested_at, after_seconds)? { + continue; + } + let identity = serde_json::to_vec(&( + &key, + &record.target_database, + &record.target_schema, + &record.target_name, + &record.data_version_hash, + after_seconds, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &sha256_hex(&identity)).await?; + run_ids.push(run_id); + } + cursor.custom = next; + Ok((transaction, (cursor, run_ids))) +} + async fn prior_start( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs index 87d76698ad..06201978b9 100644 --- a/crates/sqlbuild-orchestration/src/sensors/constants.rs +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -3,4 +3,5 @@ pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; pub(crate) const CUSTOM_SENSOR_RECHECK_SECONDS: i64 = 1; pub(crate) const SOURCE_NODE_KIND: &str = "source"; +pub(crate) const TIMESTAMP_VALUE_KIND: &str = "timestamp"; pub const MAX_SENSOR_RUN_REQUESTS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs index 8224fc177c..7ba89f5f56 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -22,7 +22,8 @@ pub(crate) fn validate_sensors( && !selection.is_empty() && selection.iter().all(|key| input.nodes.contains_key(key)) } - SensorCondition::SourceNewData { selection } => { + SensorCondition::SourceNewData { selection } + | SensorCondition::SourceStale { selection, .. } => { !selection.is_empty() && selection.iter().all(|key| { input @@ -55,7 +56,7 @@ pub(crate) fn validate_sensors( && !definitions.groups.contains_key(&sensor.job_key)) { return Err(ControlError::Invalid(format!( - "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes; source_new_data must select nonempty snapshot sources" + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes; source_new_data and source_stale must select nonempty snapshot sources" ))); } } diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index b53adb7a29..6c335bbd4b 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -48,6 +48,11 @@ pub enum SensorCondition { SourceNewData { selection: BTreeSet, }, + SourceStale { + selection: BTreeSet, + #[schema(value_type = u32, minimum = 1)] + after_seconds: NonZeroU32, + }, Custom { function: String, code_hash: String, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 1ff60fe91c..72ca127b71 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -32,6 +32,7 @@ pub(crate) const WAREHOUSE_JANITOR_EVENTS: &str = include_str!("../warehouse_jan pub(crate) const WAREHOUSE_MIGRATIONS: &str = include_str!("../warehouse_migrations.sql"); pub(crate) const WAREHOUSE_MICROBATCHES: &str = include_str!("../warehouse_microbatches.sql"); pub(crate) const SOURCE_SENSORS: &str = include_str!("../source_sensors.sql"); +pub(crate) const SOURCE_STALENESS: &str = include_str!("../source_staleness.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 54bc4c78da..f2960d0f84 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -7,9 +7,10 @@ use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, SOURCE_SENSORS, WAREHOUSE_AUDIT_RESULTS, - WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, - WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, verify_migrations, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, SOURCE_SENSORS, SOURCE_STALENESS, + WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, + WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, + verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -58,6 +59,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_migrations", WAREHOUSE_MIGRATIONS), ("warehouse_microbatches", WAREHOUSE_MICROBATCHES), ("source_sensors", SOURCE_SENSORS), + ("source_staleness", SOURCE_STALENESS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/source_staleness.sql b/crates/sqlbuild-orchestration/src/store/source_staleness.sql new file mode 100644 index 0000000000..22fc26648e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/source_staleness.sql @@ -0,0 +1,5 @@ +CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), observed_at DESC, event_id DESC +); diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 61472d4374..4d35d309f9 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -124,6 +124,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP INDEX sqb_freshness_source_latest", + "DELETE FROM sqb_schema_migrations WHERE name='source_staleness'", "DROP INDEX recall_warehouse_sources", "DELETE FROM sqb_schema_migrations WHERE name='source_sensors'", "DROP TABLE sqb_warehouse_microbatches", diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 7a86cc8e4d..5c9433e1c6 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -2,6 +2,7 @@ use std::time::Duration; +use chrono::{FixedOffset, TimeDelta}; use serde_json::json; use uuid::Uuid; @@ -172,6 +173,8 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .await .expect("pre-sensor predecessor"); for sql in [ + "DROP INDEX sqb_freshness_source_latest", + "DELETE FROM sqb_schema_migrations WHERE name='source_staleness'", "DELETE FROM recall_events WHERE producer='warehouse' AND event_type='warehouse_source_freshness'", "DROP INDEX recall_warehouse_sources", "DELETE FROM sqb_schema_migrations WHERE name='source_sensors'", @@ -377,3 +380,240 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ }); } } + +#[test] +fn given_timestamp_observations_when_age_crosses_the_boundary_then_scoped_pages_queue_each_version_once() + { + let test_cases = [ + SensorTestCase { + description: "SQLite timestamp age", + backend: Backend::Sqlite, + expected_runs: 4, + }, + SensorTestCase { + description: "PostgreSQL timestamp age", + backend: Backend::Postgres, + expected_runs: 4, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let data: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json", + )) + .expect("DuckShop stale-source fixture"), + ) + .expect("fixture"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let keys: Vec = (0..data["source_count"].as_u64().expect("source count")) + .map(|index| format!("source:raw_orders_{index:03}")) + .collect(); + for key in &keys { + input.nodes.insert( + key.clone(), + SnapshotNode { + kind: "source".into(), + definition: data["source_definitions"].get(key).cloned().unwrap_or_else( + || json!({"database": null, "schema": "main", "table": "raw_orders"}), + ), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ); + } + input.documents.insert("triggers".into(), json!({"stale_orders": { + "description": "Refresh stale timestamp sources", "job_key": "orders", + "when": {"kind": "source_stale", "selection": keys, "after_seconds": data["after_seconds"]}, + "minimum_interval_seconds": 1 + }})); + publish_snapshot(store, &input).await.expect("publish"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let first = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluation") + .expect("due"); + let mut records = Vec::new(); + for (ordinal, case) in data["cases"].as_array().expect("cases").iter().enumerate() { + let index = case["index"].as_u64().expect("index"); + let instant = first.requested_at + - TimeDelta::seconds(case["age_seconds"].as_i64().expect("age")); + let version = case["data_version"] + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| { + instant + .with_timezone( + &FixedOffset::east_opt( + case["offset_seconds"].as_i64().unwrap_or(0) as i32, + ) + .expect("offset"), + ) + .format(case["format"].as_str().unwrap_or("%Y-%m-%dT%H:%M:%S%.f%:z")) + .to_string() + }); + let record = WarehouseFreshnessRecord { + event_id: Some(format!("stale-source-{ordinal:03}")), + state_database: None, + state_schema: "main".into(), + source_name: format!("raw_orders_{index:03}"), + target_database: case["target_database"].as_str().map(str::to_owned), + target_schema: Some(case["target_schema"].as_str().unwrap_or("main").into()), + target_name: Some(case["target_name"].as_str().unwrap_or("raw_orders").into()), + run_id: "external-orders".into(), + strategy: "max_column".into(), + value_kind: case["value_kind"].as_str().unwrap_or("timestamp").into(), + data_version: (!case["null"].as_bool().unwrap_or(false)).then_some(version), + data_version_hash: format!("orders-version-{index}"), + observed_at: first.requested_at + + TimeDelta::microseconds( + case["observed_offset_microseconds"].as_i64().unwrap_or(0), + ), + }; + records.push(record); + } + let batch = SourceFreshnessBatch { + session_id, + generation: 1, + records, + reconciliation: None, + }; + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("mirror observations"); + let fault = TransactionFault { + description: "stale cursor rollback", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected stale cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected stale cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &first) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback") + .is_empty() + ); + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &first), + ) + .await + .expect("bounded first page") + .expect("complete"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("first page") + .len(), + 1, + "{}: strict age threshold and current source binding", + test_case.description + ); + for page in data["later_pages"].as_array().expect("page outcomes") { + tokio::time::sleep(Duration::from_secs(1)).await; + let request = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next evaluation") + .expect("due"); + assert_eq!( + request.cursor.custom.as_deref(), + page["expected_cursor"].as_str(), + "{}", + test_case.description + ); + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &request), + ) + .await + .expect("bounded indexed source lookup") + .expect("complete"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("queued runs") + .len(), + page["expected_runs"].as_u64().expect("run count") as usize, + "{}", + test_case.description + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &request) + .await + .expect("lost acknowledgement"), + 0 + ); + } + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("clear mirror"), + "DELETE FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("delete disposable mirror"); + commit(transaction).await.expect("commit clear"); + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("rebuild mirror"); + tokio::time::sleep(Duration::from_secs(1)).await; + let request = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("rebuilt evaluation") + .expect("due"); + complete_builtin_sensor(store, &lease, &request) + .await + .expect("replay stale conditions"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("deduplicated requests") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 1ece736b2c..11a7038c12 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP INDEX sqb_freshness_source_latest", + "DELETE FROM sqb_schema_migrations WHERE name='source_staleness'", "DROP INDEX recall_warehouse_sources", "DELETE FROM sqb_schema_migrations WHERE name='source_sensors'", "DROP TABLE sqb_warehouse_microbatches", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 74e15e69db..13b4431c30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -5,6 +5,7 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; +pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; /// The sensor context is supplied by the host, not a provider. pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 7b3acd2849..eb99a77f40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -58,6 +58,8 @@ pub(crate) struct AuthoredSensorCondition { pub(crate) executed_successfully: Option>, #[pyo3(default)] pub(crate) select: Option, + #[pyo3(default, from_py_with = authored_setting)] + pub(crate) after_seconds: Value, } #[derive(FromPyObject)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 519768dd66..98afe8125a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, wit use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, + CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -279,7 +279,9 @@ pub(crate) fn compile_sensors( let condition = match (sensor.when, sensor.function) { (Some(when), None) => { let mut condition = json!({"kind": when.kind}); - if when.kind != SOURCE_NEW_DATA_CONDITION { + if when.kind == SOURCE_STALE_CONDITION { + condition["after_seconds"] = when.after_seconds; + } else if when.kind != SOURCE_NEW_DATA_CONDITION { condition["kinds"] = json!(when.kinds); } if let Some(job) = when.job { @@ -333,10 +335,12 @@ fn sensor_nodes(graph: &ProjectGraph, selectors: &[String], field: &str) -> PyRe "{field} needs nonempty selectors" ))); } - if field == SOURCE_NEW_DATA_CONDITION && nodes.iter().any(|key| !key.starts_with("source:")) { - return Err(PyValueError::new_err( - "source_new_data must select only source nodes", - )); + if matches!(field, SOURCE_NEW_DATA_CONDITION | SOURCE_STALE_CONDITION) + && nodes.iter().any(|key| !key.starts_with("source:")) + { + return Err(PyValueError::new_err(format!( + "{field} must select only source nodes" + ))); } Ok(json!(nodes)) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2ee216eef3..93e5d60093 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1740,7 +1740,7 @@ class NativeCompiledAudit: def meta(self) -> dict[str, object] | None: ... class NativeOrchestrationSnapshot: - """Compiled jobs, schedules and event/source-version sensors retained as publishable data.""" + """Compiled jobs, schedules and event/source-version/age sensors as publishable data.""" def to_json(self) -> str: ... def publish(self, server_url: str, bearer: str) -> str: ... diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index c176f6f09a..e07a3119cf 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -14,6 +14,7 @@ Sensor, SkipReason, SourceNewDataCondition, + SourceStaleCondition, ) from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule as calendar_schedule, @@ -30,6 +31,9 @@ from sqlbuild.orchestration_declarations.models.authoring import ( SourceNewDataCondition as source_new_data, ) +from sqlbuild.orchestration_declarations.models.authoring import ( + SourceStaleCondition as source_stale, +) from sqlbuild.orchestration_declarations.models.sensor_context import SensorContext from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind @@ -50,6 +54,7 @@ "SensorContext", "SkipReason", "SourceNewDataCondition", + "SourceStaleCondition", "calendar_schedule", "compile_snapshot", "job", @@ -60,4 +65,5 @@ "schedule", "sensor", "source_new_data", + "source_stale", ) diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index fc5fcca6cb..ce783dda20 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -117,6 +117,15 @@ class SourceNewDataCondition: kind: Literal["source_new_data"] = field(default="source_new_data", init=False) +@dataclass(frozen=True, kw_only=True) +class SourceStaleCondition: + """Match timestamp source versions older than after_seconds, once per version.""" + + select: str + after_seconds: int + kind: Literal["source_stale"] = field(default="source_stale", init=False) + + @dataclass(frozen=True, eq=False) class Sensor: """Start a job for each matching event; event identity deduplicates replayed requests.""" @@ -128,6 +137,7 @@ class Sensor: | RunEventCondition | NodeEventCondition | SourceNewDataCondition + | SourceStaleCondition | None ) = field(default=None, kw_only=True) run: Job | JobGroup = field(kw_only=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py index 7bf8f09885..cebf53dfe5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -47,7 +47,29 @@ "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, - ) + ), + WorkerExecutionTestCase( + "stale external timestamp versions", + "SELECT 73 AS amount", + "SELECT amount FROM orders", + "success", + [(73,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml" + ).read_text(), + "python/loaders/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/orders_loader.py" + ).read_text(), + "source_observations.json": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json" + ).read_text(), + }, + ), ], ids=lambda case: case.description, ) @@ -55,7 +77,12 @@ def test_given_external_source_versions_when_a_location_resyncs_then_only_change test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: data: dict[str, Any] = json.loads( - Path("benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json").read_text() + test_case.files.get( + "source_observations.json", + Path( + "benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json" + ).read_text(), + ) ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: for phase, (source, count) in enumerate( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 64638b3774..56c4580a8c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -762,6 +762,31 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="source_stale freezes source nodes and a positive timestamp-age threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_jobs=("orders_refresh",), + expected_sensors={ + "on_stale_orders": { + "description": "Refresh orders once for each timestamp version older than one minute.", + "job_key": "orders_refresh", + "when": { + "kind": "source_stale", + "selection": ["source:raw_orders"], + "after_seconds": 60, + }, + "minimum_interval_seconds": 1, + "default_status": "running", + } + }, + ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", files={ @@ -1035,6 +1060,63 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="source_new_data needs nonempty selectors", ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects a zero threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace("after_seconds=60", "after_seconds=0"), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_error=ValueError, + expected_message="sensor on_stale_orders", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects a boolean threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace("after_seconds=60", "after_seconds=True"), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_error=ValueError, + expected_message="sensor on_stale_orders", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects an overflowing threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace("after_seconds=60", "after_seconds=2**32"), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_error=ValueError, + expected_message="sensor on_stale_orders", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects model selections", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace('select="source:raw_orders"', 'select="orders"'), + }, + expected_error=ValueError, + expected_message="source_stale must select only source nodes", + ), SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", files={ From e5b56ed656d2f3e08afd2f3239521e7dc10c666d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:39:41 +0000 Subject: [PATCH 096/138] perf: avoid reimporting committed reconciliation pages --- .../src/warehouse_metadata/_helpers/events.rs | 105 +++++++++--------- .../_helpers/reconciliation.rs | 7 +- .../main/ingest_fingerprints.rs | 87 ++++++++------- .../main/ingest_source_freshness.rs | 78 +++++++------ 4 files changed, 144 insertions(+), 133 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs index 546f01308a..e57b0c77c5 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -38,8 +38,9 @@ pub(crate) async fn ingest_events( (batch.session_id, batch.generation), ) .await?; + let mut received = false; if let Some(page) = batch.reconciliation { - transaction = begin_reconciliation( + (transaction, received) = begin_reconciliation( transaction, (table, location_id, target), page, @@ -49,57 +50,59 @@ pub(crate) async fn ingest_events( } let mut applied = false; let mut keys: Vec = Vec::new(); - for record in batch.records { - let event_id = scoped_event_id( - (location_id, target), - ( - record.location.database.as_deref(), - &record.location.schema, - &source_table, - ), - &record.source_id, - )?; - if batch.reconciliation.is_some() { - keys.push(event_id); + if !received { + for record in batch.records { + let event_id = scoped_event_id( + (location_id, target), + ( + record.location.database.as_deref(), + &record.location.schema, + &source_table, + ), + &record.source_id, + )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } + let mut columns = vec![ + "event_id", + "location_id", + "target", + "source_event_id", + "observed_at", + "state_database", + "state_schema", + ]; + let mut parameters = vec![ + Parameter::Id(event_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(record.source_id), + Parameter::Time(record.observed_at), + Parameter::OptionalText(record.location.database), + Parameter::Text(record.location.schema), + ]; + for (column, value) in record.columns { + columns.push(column); + parameters.push(value); + } + columns.extend(["record_json", "received_at"]); + parameters.extend([ + Parameter::Json(record.record_json), + Parameter::Time(transaction.now), + ]); + let placeholders = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let sql = format!( + "INSERT INTO {table} ({}) VALUES ({placeholders}) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + columns.join(",") + ); + let inserted; + (transaction, inserted) = query(transaction, &sql, parameters).await?; + applied |= !inserted.is_empty(); } - let mut columns = vec![ - "event_id", - "location_id", - "target", - "source_event_id", - "observed_at", - "state_database", - "state_schema", - ]; - let mut parameters = vec![ - Parameter::Id(event_id), - Parameter::Id(location_id), - Parameter::Text(target.into()), - Parameter::Text(record.source_id), - Parameter::Time(record.observed_at), - Parameter::OptionalText(record.location.database), - Parameter::Text(record.location.schema), - ]; - for (column, value) in record.columns { - columns.push(column); - parameters.push(value); - } - columns.extend(["record_json", "received_at"]); - parameters.extend([ - Parameter::Json(record.record_json), - Parameter::Time(transaction.now), - ]); - let placeholders = (1..=parameters.len()) - .map(|index| format!("${index}")) - .collect::>() - .join(","); - let sql = format!( - "INSERT INTO {table} ({}) VALUES ({placeholders}) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", - columns.join(",") - ); - let inserted; - (transaction, inserted) = query(transaction, &sql, parameters).await?; - applied |= !inserted.is_empty(); } commit_ingestion( transaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs index 9c587565fb..a3c2265bfc 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs @@ -13,12 +13,13 @@ use crate::warehouse_metadata::models::{ WarehouseReconciliationCursor, WarehouseReconciliationPage, WarehouseSyncResponse, }; +/// A matching receipt proves its records and scan keys committed in the same transaction. pub(crate) async fn begin_reconciliation( mut transaction: ControlTransaction, scope: (&str, Uuid, &str), page: &WarehouseReconciliationPage, mut batch: Value, -) -> Result { +) -> Result<(ControlTransaction, bool), ControlError> { if page.state_schema.trim().is_empty() || page.day.is_some_and(|day| day.succ_opt().is_none()) || (!page.complete && page.after.is_some()) @@ -76,7 +77,7 @@ pub(crate) async fn begin_reconciliation( "reconciliation page retry changed its content".into(), )); } - return Ok(transaction); + return Ok((transaction, true)); } let expected = latest.first().map_or(Some(0), |row| { row["page"].as_u64().and_then(|page| page.checked_add(1)) @@ -96,7 +97,7 @@ pub(crate) async fn begin_reconciliation( "INSERT INTO sqb_warehouse_scan_pages (scan_id,page,scope_hash,request_hash,complete,received_at) VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(page.scan_id), Parameter::Integer(i64::from(page.page)), Parameter::Text(scope_hash), Parameter::Text(request_hash), Parameter::Boolean(page.complete), Parameter::Time(now)]).await?; - Ok(transaction) + Ok((transaction, false)) } /// Each request visits at most one mirror page, including retained and concurrently received rows. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs index fc7a2a0d1e..150d27f615 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs @@ -26,8 +26,9 @@ pub async fn ingest_fingerprints( (batch.session_id, batch.generation), ) .await?; + let mut received = false; if let Some(page) = &batch.reconciliation { - transaction = begin_reconciliation( + (transaction, received) = begin_reconciliation( transaction, ("sqb_warehouse_fingerprints", location_id, target), page, @@ -37,52 +38,54 @@ pub async fn ingest_fingerprints( } let mut applied = false; let mut keys: Vec = Vec::new(); - for record in &batch.records { - let source_event_id = source_event_id(record); - let event_id = scoped_event_id( - (location_id, target), - ( - record.state_database.as_deref(), - &record.state_schema, - "_sqlbuild_fingerprints", - ), - &source_event_id, - )?; - if batch.reconciliation.is_some() { - keys.push(event_id); - } - let existing; - (transaction, existing) = query( - transaction, - "SELECT event_id FROM sqb_warehouse_fingerprints WHERE event_id=$1", - vec![Parameter::Id(event_id)], - ) - .await?; - if !existing.is_empty() { - continue; - } - let definition_text_hash = sha256_hex(record.definition.as_bytes()); - let mut record_json = json!(record); - if let serde_json::Value::Object(object) = &mut record_json { - object.remove("definition"); - } - let received_at = transaction.now; - transaction = store_source_text( - transaction, - ( - definition_text_hash.clone(), - compress(record.definition.as_bytes())?, - ), - ) - .await?; - let inserted; - (transaction, inserted) = query(transaction, + if !received { + for record in &batch.records { + let source_event_id = source_event_id(record); + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_fingerprints", + ), + &source_event_id, + )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } + let existing; + (transaction, existing) = query( + transaction, + "SELECT event_id FROM sqb_warehouse_fingerprints WHERE event_id=$1", + vec![Parameter::Id(event_id)], + ) + .await?; + if !existing.is_empty() { + continue; + } + let definition_text_hash = sha256_hex(record.definition.as_bytes()); + let mut record_json = json!(record); + if let serde_json::Value::Object(object) = &mut record_json { + object.remove("definition"); + } + let received_at = transaction.now; + transaction = store_source_text( + transaction, + ( + definition_text_hash.clone(), + compress(record.definition.as_bytes())?, + ), + ) + .await?; + let inserted; + (transaction, inserted) = query(transaction, "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema) \ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; - applied |= !inserted.is_empty(); + applied |= !inserted.is_empty(); + } } commit_ingestion( transaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs index 173f08a11a..3db9bf359e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -27,8 +27,9 @@ pub async fn ingest_source_freshness( (batch.session_id, batch.generation), ) .await?; + let mut received = false; if let Some(page) = &batch.reconciliation { - transaction = begin_reconciliation( + (transaction, received) = begin_reconciliation( transaction, ("sqb_warehouse_freshness", location_id, target), page, @@ -38,47 +39,50 @@ pub async fn ingest_source_freshness( } let mut applied = false; let mut keys: Vec = Vec::new(); - for record in &batch.records { - let source_event_id = record.event_id.clone().unwrap_or_else(|| { - legacy_event_id(&FreshnessEventIdentity { - source_name: &record.source_name, - target_database: record.target_database.as_deref(), - target_schema: record.target_schema.as_deref(), - target_name: record.target_name.as_deref(), - run_id: &record.run_id, - strategy: &record.strategy, - value_kind: &record.value_kind, - data_version_hash: &record.data_version_hash, - observed_at: &record - .observed_at - .format("%Y-%m-%dT%H:%M:%S%.6f+00:00") - .to_string(), - }) - }); - let event_id = scoped_event_id( - (location_id, target), - ( - record.state_database.as_deref(), - &record.state_schema, - "_sqlbuild_source_freshness", - ), - &source_event_id, - )?; - if batch.reconciliation.is_some() { - keys.push(event_id); - } - let received_at = transaction.now; - let inserted; - (transaction, inserted) = query(transaction, + if !received { + for record in &batch.records { + let source_event_id = record.event_id.clone().unwrap_or_else(|| { + legacy_event_id(&FreshnessEventIdentity { + source_name: &record.source_name, + target_database: record.target_database.as_deref(), + target_schema: record.target_schema.as_deref(), + target_name: record.target_name.as_deref(), + run_id: &record.run_id, + strategy: &record.strategy, + value_kind: &record.value_kind, + data_version_hash: &record.data_version_hash, + observed_at: &record + .observed_at + .format("%Y-%m-%dT%H:%M:%S%.6f+00:00") + .to_string(), + }) + }); + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_source_freshness", + ), + &source_event_id, + )?; + if batch.reconciliation.is_some() { + keys.push(event_id); + } + let received_at = transaction.now; + let inserted; + (transaction, inserted) = query(transaction, "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at,state_database,state_schema) \ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; - if !inserted.is_empty() { - transaction = - append_observation(transaction, (location_id, target, event_id), record).await?; - applied = true; + if !inserted.is_empty() { + transaction = + append_observation(transaction, (location_id, target, event_id), record) + .await?; + applied = true; + } } } commit_ingestion( From 6d5ddd1247fb54fd43dabf4c1680db47d78be678 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:29:10 +0000 Subject: [PATCH 097/138] feat: export security audit facts through an independent webhook --- .../security_audit_delivery.json | 7 + .../src/native_commands/_helpers/execution.rs | 46 ++-- .../src/workers/_helpers/invocation.rs | 2 + .../src/authorization/main/mod.rs | 1 + .../authorization/main/read_audit_events.rs | 25 +++ .../authorization/main/read_security_audit.rs | 16 +- crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/platform/_helpers/mod.rs | 1 - .../src/platform/constants.rs | 2 - .../src/platform/main/mod.rs | 2 - .../src/platform/models.rs | 16 -- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 9 +- .../src/store/security_audit_sink.sql | 7 + .../_helpers/delivery.rs} | 43 ++-- .../src/webhooks/_helpers/events.rs | 34 +++ .../src/webhooks/_helpers/mod.rs | 2 + .../src/webhooks/constants.rs | 7 + .../main/connect_webhook.rs} | 19 +- .../src/webhooks/main/mod.rs | 2 + .../main/run_webhook.rs} | 20 +- .../src/webhooks/mod.rs | 6 + .../src/webhooks/models.rs | 61 +++++ .../tests/credentials/test_audit.rs | 211 +++++++++++++++++- .../tests/credentials/test_types.rs | 10 + .../tests/platform/test_webhook.rs | 29 +-- .../tests/sensors/test_sources.rs | 2 + .../tests/store/helpers.rs | 2 + fensu.toml | 6 +- .../publication/docker/helpers.py | 2 + .../compiled_project/publication/helpers.py | 6 + .../compiled_project/publication/test_dev.py | 6 +- .../publication/test_docker_worker.py | 3 +- 33 files changed, 499 insertions(+), 108 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs create mode 100644 crates/sqlbuild-orchestration/src/store/security_audit_sink.sql rename crates/sqlbuild-orchestration/src/{platform/_helpers/webhook_delivery.rs => webhooks/_helpers/delivery.rs} (55%) create mode 100644 crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs create mode 100644 crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/webhooks/constants.rs rename crates/sqlbuild-orchestration/src/{platform/main/connect_platform_webhook.rs => webhooks/main/connect_webhook.rs} (52%) create mode 100644 crates/sqlbuild-orchestration/src/webhooks/main/mod.rs rename crates/sqlbuild-orchestration/src/{platform/main/run_platform_sink.rs => webhooks/main/run_webhook.rs} (61%) create mode 100644 crates/sqlbuild-orchestration/src/webhooks/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/webhooks/models.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json b/benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json new file mode 100644 index 0000000000..cde7c2ee40 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json @@ -0,0 +1,7 @@ +{ + "events": 105, + "action": "role.assigned", + "scope": {"location": "orders", "target": "development"}, + "before": {"role": null}, + "after": {"role": "viewer", "note": "Order metadata — read only"} +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 78b2666cc3..9f3bb587ff 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -15,9 +15,7 @@ use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, D use sqlbuild_orchestration::authorization::main::recover_administrator::recover_administrator; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; -use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; -use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink; use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::{ LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, @@ -36,6 +34,11 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryKind, WarehouseHistoryPage, WarehouseReconciliationCursor, WarehouseStateLocation, WarehouseSyncResponse, }; +use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::webhooks::main::{ + connect_webhook::connect_webhook, run_webhook::run_webhook, +}; +use sqlbuild_orchestration::webhooks::models::{Webhook, WebhookSource}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; use tokio::process::{Child, Command}; use tokio::sync::{mpsc, oneshot, watch}; @@ -176,6 +179,7 @@ fn development_location_role( .env("SQLBUILD_LOCATION_TOKEN", "local-development") .env_remove("SQLBUILD_CONTROL_DATABASE_URL") .env_remove("SQLBUILD_PLATFORM_WEBHOOK_URL") + .env_remove(SECURITY_AUDIT_WEBHOOK_VARIABLE) .env_remove("SQLBUILD_RUN_TOKEN") .env_remove("SQLBUILD_DEVELOPMENT_CONTROL_FILE") .stdin(Stdio::null()) @@ -759,15 +763,29 @@ pub(crate) fn start_platform_roles( tasks: &mut JoinSet>, (store, shutdown): (&ControlStore, &watch::Receiver), ) -> Result<(), ControlError> { - let sink = match std::env::var(PLATFORM_WEBHOOK_VARIABLE) { - Ok(endpoint) => Some(connect_platform_webhook(&endpoint)?), - Err(std::env::VarError::NotPresent) => None, - Err(_) => { - return Err(ControlError::Invalid( - "platform webhook configuration must be UTF-8".into(), - )); + let mut sinks: Vec<(Webhook, &str)> = Vec::new(); + for (variable, source, label) in [ + ( + PLATFORM_WEBHOOK_VARIABLE, + WebhookSource::Platform, + "Platform", + ), + ( + SECURITY_AUDIT_WEBHOOK_VARIABLE, + WebhookSource::SecurityAudit, + "Security audit", + ), + ] { + match std::env::var(variable) { + Ok(endpoint) => sinks.push((connect_webhook(&endpoint, source)?, label)), + Err(std::env::VarError::NotPresent) => {} + Err(_) => { + return Err(ControlError::Invalid(format!( + "{label} webhook configuration must be UTF-8" + ))); + } } - }; + } drop(write_output(&[( Stream::Stderr, "Starting platform watchdog...\n", @@ -781,17 +799,17 @@ pub(crate) fn start_platform_roles( Stream::Stderr, "Platform watchdog started\n", )])); - if let Some(sink) = sink { + for (sink, label) in sinks { drop(write_output(&[( Stream::Stderr, - "Starting platform webhook...\n", + &format!("Starting {label} webhook...\n"), )])); let sink_store = store.clone(); let sink_shutdown = shutdown.clone(); - tasks.spawn(async move { run_platform_sink(&sink_store, sink, sink_shutdown).await }); + tasks.spawn(async move { run_webhook(&sink_store, sink, sink_shutdown).await }); drop(write_output(&[( Stream::Stderr, - "Platform webhook started\n", + &format!("{label} webhook started\n"), )])); } Ok(()) diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs index a951cc74d6..f1721202a0 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -3,6 +3,7 @@ use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; use crate::workers::models::LauncherScope; @@ -28,6 +29,7 @@ pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { name == control_variable || name == PLATFORM_WEBHOOK_VARIABLE + || name == SECURITY_AUDIT_WEBHOOK_VARIABLE || matches!( name, "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs index 36969020a7..6d5a0e05b9 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -3,5 +3,6 @@ pub(crate) mod authorize_session; pub mod bootstrap_user; pub mod change_administrator; pub(crate) mod deny_authorization; +pub(crate) mod read_audit_events; pub mod read_security_audit; pub mod recover_administrator; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs b/crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs new file mode 100644 index 0000000000..8791f4349e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs @@ -0,0 +1,25 @@ +//! Authorized server consumers page security facts independently of clocks and transport. + +use crate::credentials::models::SecurityAuditEvent; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_audit_events( + transaction: ControlTransaction, + (after, limit): (i64, u32), +) -> Result<(ControlTransaction, Vec), ControlError> { + if after < 0 || !(1..=100).contains(&limit) { + return Err(ControlError::Invalid( + "audit pages require a nonnegative storage cursor and a limit between 1 and 100".into(), + )); + } + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,actor,action,granting_role,request_id,ip_address,scope_json,before_json,after_json \ + FROM sqb_security_audit WHERE storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(i64::from(limit))]).await?; + Ok(( + transaction, + rows.into_iter().map(decode_row).collect::>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs b/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs index d03d190806..84cd1bbbc1 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs @@ -1,12 +1,13 @@ //! Live installation-wide audit permission and each bounded page share one transaction. use crate::authorization::main::authorize_session::authorize_session; +use crate::authorization::main::read_audit_events::read_audit_events; use crate::credentials::models::{ Action, AuditContext, DefaultRole, PermissionScope, SecurityAuditEvent, }; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn read_security_audit( store: &ControlStore, @@ -25,16 +26,7 @@ pub async fn read_security_audit( DefaultRole::None, ) .await?; - if after < 0 || !(1..=100).contains(&limit) { - return Err(ControlError::Invalid( - "audit pages require a nonnegative storage cursor and a limit between 1 and 100".into(), - )); - } - let (transaction, rows) = query(transaction, - "SELECT storage_id,event_id,occurred_at,actor,action,granting_role,request_id,ip_address,scope_json,before_json,after_json \ - FROM sqb_security_audit WHERE storage_id>$1 ORDER BY storage_id LIMIT $2", - vec![Parameter::Integer(after), Parameter::Integer(i64::from(limit))]).await?; - let events = rows.into_iter().map(decode_row).collect::>()?; + let (transaction, events) = read_audit_events(transaction, (after, limit)).await?; commit(transaction).await?; Ok(events) } diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 80ef365a70..87aebeb15e 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -13,4 +13,5 @@ pub mod sensors; pub mod snapshots; pub mod store; pub mod warehouse_metadata; +pub mod webhooks; pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index fddc0b0e5a..cadc2fcacd 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -2,5 +2,4 @@ pub(crate) mod liveness; pub(crate) mod location_heartbeat; pub(crate) mod observations; pub(crate) mod watchdog; -pub(crate) mod webhook_delivery; pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index 75d8a07df7..061b721ebf 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -4,5 +4,3 @@ pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; -pub(crate) const SINK_REQUEST_TIMEOUT_SECONDS: u64 = 10; -pub(crate) const SINK_POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 8124f49518..0cec22828b 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,7 +1,6 @@ pub mod claim_location_work; pub mod complete_location_work; pub mod connect_location_work; -pub mod connect_platform_webhook; pub mod location_health; pub mod location_work; pub(crate) mod read_location_work; @@ -10,5 +9,4 @@ pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; pub(crate) mod require_instance; -pub mod run_platform_sink; pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index fc5bf7a971..cd79677e03 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -228,22 +228,6 @@ pub(crate) struct WatchdogLocation { pub(crate) last_seen_at: DateTime, } -/// The endpoint may contain credentials; neither errors nor Debug reveal it. -#[derive(Clone)] -pub struct PlatformWebhook { - pub(crate) client: reqwest::Client, - pub(crate) url: reqwest::Url, - pub(crate) sink_id: String, -} - -impl std::fmt::Debug for PlatformWebhook { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("PlatformWebhook") - .finish_non_exhaustive() - } -} - /// Stable event identity permits receiver-side deduplication after uncertain acknowledgements. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PlatformEventEnvelope { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 72ca127b71..3d3bff65c6 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -33,6 +33,7 @@ pub(crate) const WAREHOUSE_MIGRATIONS: &str = include_str!("../warehouse_migrati pub(crate) const WAREHOUSE_MICROBATCHES: &str = include_str!("../warehouse_microbatches.sql"); pub(crate) const SOURCE_SENSORS: &str = include_str!("../source_sensors.sql"); pub(crate) const SOURCE_STALENESS: &str = include_str!("../source_staleness.sql"); +pub(crate) const SECURITY_AUDIT_SINK: &str = include_str!("../security_audit_sink.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index f2960d0f84..260219a98e 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -7,10 +7,10 @@ use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SENSORS, SOURCE_SENSORS, SOURCE_STALENESS, - WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, - WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, - verify_migrations, + PLATFORM_SINK, RUN_SENSORS, SCHEMA, SECURITY_AUDIT_SINK, SENSORS, SOURCE_SENSORS, + SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, + WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, + WAREHOUSE_RECONCILIATION, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -60,6 +60,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_microbatches", WAREHOUSE_MICROBATCHES), ("source_sensors", SOURCE_SENSORS), ("source_staleness", SOURCE_STALENESS), + ("security_audit_sink", SECURITY_AUDIT_SINK), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/security_audit_sink.sql b/crates/sqlbuild-orchestration/src/store/security_audit_sink.sql new file mode 100644 index 0000000000..ff34df7007 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/security_audit_sink.sql @@ -0,0 +1,7 @@ +CREATE TABLE sqb_security_audit_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at {time} NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/delivery.rs similarity index 55% rename from crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs rename to crates/sqlbuild-orchestration/src/webhooks/_helpers/delivery.rs index eb5547281b..1e39cc0409 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/delivery.rs @@ -1,54 +1,63 @@ //! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. -use crate::platform::main::read_platform_events::read_platform_events; -use crate::platform::models::PlatformWebhook; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::webhooks::_helpers::events::read_events; +use crate::webhooks::models::Webhook; -/// A lost acknowledgement may resend the same event_id; receivers must deduplicate by that ID. -pub(crate) async fn deliver_platform_events( +/// A lost acknowledgement may resend the same event_id; receivers deduplicate by that ID. +pub(crate) async fn deliver_webhook( store: &ControlStore, - sink: &PlatformWebhook, + sink: &Webhook, ) -> Result { let transaction = begin(store, TransactionMode::Read).await?; + let table = sink.source.deliveries(); let (transaction, rows) = query( transaction, - "SELECT COALESCE(MAX(storage_id), 0) AS cursor FROM sqb_platform_sink_deliveries WHERE sink_id=$1", + &format!("SELECT COALESCE(MAX(storage_id), 0) AS cursor FROM {table} WHERE sink_id=$1"), vec![Parameter::Text(sink.sink_id.clone())], ) .await?; let cursor = serde_json::from_value(rows[0]["cursor"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; - let (transaction, events) = read_platform_events(transaction, cursor).await?; + let (transaction, events) = read_events(transaction, sink.source, cursor).await?; commit(transaction).await?; let count = events.len(); for event in events { + let (storage_id, event_id) = event.identity(); let response = sink .client .post(sink.url.clone()) - .header("Idempotency-Key", event.event_id.to_string()) + .header("Idempotency-Key", event_id.to_string()) .json(&event) .send() .await - .map_err(|_| { - ControlError::Invalid("platform webhook delivery failed; event retained".into()) - })?; + .map_err(|_| ControlError::Invalid("webhook delivery failed; event retained".into()))?; if !response.status().is_success() { return Err(ControlError::Invalid(format!( - "platform webhook returned HTTP {}; event retained", + "webhook returned HTTP {}; event retained", response.status().as_u16() ))); } let transaction = begin(store, TransactionMode::Write).await?; let now = transaction.now; - let (transaction, _) = query(transaction, - "INSERT INTO sqb_platform_sink_deliveries (sink_id, storage_id, event_id, delivered_at) \ - VALUES ($1,$2,$3,$4) ON CONFLICT (sink_id, storage_id) DO NOTHING", - vec![Parameter::Text(sink.sink_id.clone()), Parameter::Integer(event.storage_id), - Parameter::Text(event.event_id.to_string()), Parameter::Time(now)]).await?; + let (transaction, _) = query( + transaction, + &format!( + "INSERT INTO {table} (sink_id, storage_id, event_id, delivered_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (sink_id, storage_id) DO NOTHING" + ), + vec![ + Parameter::Text(sink.sink_id.clone()), + Parameter::Integer(storage_id), + Parameter::Text(event_id.to_string()), + Parameter::Time(now), + ], + ) + .await?; commit(transaction).await?; } Ok(count) diff --git a/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs new file mode 100644 index 0000000000..50c830a7a8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs @@ -0,0 +1,34 @@ +//! Source-specific readers retain authority; delivery owns only acknowledgements. + +use crate::authorization::main::read_audit_events::read_audit_events; +use crate::platform::main::read_platform_events::read_platform_events; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; +use crate::webhooks::models::{WebhookEvent, WebhookSource}; + +pub(crate) async fn read_events( + transaction: ControlTransaction, + source: WebhookSource, + after: i64, +) -> Result<(ControlTransaction, Vec), ControlError> { + match source { + WebhookSource::Platform => { + let (transaction, events) = read_platform_events(transaction, after).await?; + Ok(( + transaction, + events.into_iter().map(WebhookEvent::Platform).collect(), + )) + } + WebhookSource::SecurityAudit => { + let (transaction, events) = read_audit_events(transaction, (after, 100)).await?; + Ok(( + transaction, + events + .into_iter() + .map(Box::new) + .map(WebhookEvent::SecurityAudit) + .collect(), + )) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs new file mode 100644 index 0000000000..3fe549c1b3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod delivery; +pub(crate) mod events; diff --git a/crates/sqlbuild-orchestration/src/webhooks/constants.rs b/crates/sqlbuild-orchestration/src/webhooks/constants.rs new file mode 100644 index 0000000000..e0742aaf9f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/constants.rs @@ -0,0 +1,7 @@ +//! Delivery bounds shared by independent native event consumers. + +/// Only server operators configure this destination; never inherit it in a location or worker. +pub const SECURITY_AUDIT_WEBHOOK_VARIABLE: &str = "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL"; + +pub(crate) const REQUEST_TIMEOUT_SECONDS: u64 = 10; +pub(crate) const POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs b/crates/sqlbuild-orchestration/src/webhooks/main/connect_webhook.rs similarity index 52% rename from crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs rename to crates/sqlbuild-orchestration/src/webhooks/main/connect_webhook.rs index 4a715b9ec8..e724005a6c 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs +++ b/crates/sqlbuild-orchestration/src/webhooks/main/connect_webhook.rs @@ -4,28 +4,27 @@ use std::time::Duration; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use crate::platform::constants::SINK_REQUEST_TIMEOUT_SECONDS; -use crate::platform::models::PlatformWebhook; use crate::store::errors::ControlError; +use crate::webhooks::constants::REQUEST_TIMEOUT_SECONDS; +use crate::webhooks::models::{Webhook, WebhookSource}; /// Changing the endpoint creates a new consumer, starting from the retained event history. -pub fn connect_platform_webhook(endpoint: &str) -> Result { +pub fn connect_webhook(endpoint: &str, source: WebhookSource) -> Result { let url = reqwest::Url::parse(endpoint) - .map_err(|_| ControlError::Invalid("platform webhook needs an HTTP(S) URL".into()))?; + .map_err(|_| ControlError::Invalid("webhook needs an HTTP(S) URL".into()))?; if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { - return Err(ControlError::Invalid( - "platform webhook needs an HTTP(S) URL".into(), - )); + return Err(ControlError::Invalid("webhook needs an HTTP(S) URL".into())); } let sink_id = sha256_hex(url.as_str().as_bytes()); let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(SINK_REQUEST_TIMEOUT_SECONDS)) + .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECONDS)) .build() - .map_err(|_| ControlError::Invalid("platform webhook client could not start".into()))?; - Ok(PlatformWebhook { + .map_err(|_| ControlError::Invalid("webhook client could not start".into()))?; + Ok(Webhook { client, url, sink_id, + source, }) } diff --git a/crates/sqlbuild-orchestration/src/webhooks/main/mod.rs b/crates/sqlbuild-orchestration/src/webhooks/main/mod.rs new file mode 100644 index 0000000000..76523ab16d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connect_webhook; +pub mod run_webhook; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs b/crates/sqlbuild-orchestration/src/webhooks/main/run_webhook.rs similarity index 61% rename from crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs rename to crates/sqlbuild-orchestration/src/webhooks/main/run_webhook.rs index 02757c1627..2eeda4a457 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs +++ b/crates/sqlbuild-orchestration/src/webhooks/main/run_webhook.rs @@ -1,35 +1,35 @@ -//! Deliver platform health independently of user interpreters and scheduler leadership. +//! Deliver durable events independently of user interpreters and scheduler leadership. use std::time::Duration; use tokio::sync::watch; -use crate::platform::_helpers::webhook_delivery::deliver_platform_events; -use crate::platform::constants::SINK_POLL_SECONDS; -use crate::platform::models::PlatformWebhook; use crate::store::errors::ControlError; use crate::store::models::ControlStore; +use crate::webhooks::_helpers::delivery::deliver_webhook; +use crate::webhooks::constants::POLL_SECONDS; +use crate::webhooks::models::Webhook; -pub async fn run_platform_sink( +pub async fn run_webhook( store: &ControlStore, - sink: PlatformWebhook, + sink: Webhook, mut shutdown: watch::Receiver, ) -> Result<(), ControlError> { let mut failed = false; while !*shutdown.borrow() { tokio::select! { - result = deliver_platform_events(store, &sink) => { + result = deliver_webhook(store, &sink) => { match result { Ok(count) => { if failed { - eprintln!("Platform webhook delivery recovered"); + eprintln!("{} webhook delivery recovered", sink.source.label()); } failed = false; if count > 0 { continue; } }, Err(_) => { if !failed { - eprintln!("Platform webhook delivery failed; retaining events and retrying"); + eprintln!("{} webhook delivery failed; retaining events and retrying", sink.source.label()); } failed = true; }, @@ -40,7 +40,7 @@ pub async fn run_platform_sink( }, } tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(SINK_POLL_SECONDS)) => {}, + _ = tokio::time::sleep(Duration::from_secs(POLL_SECONDS)) => {}, result = shutdown.changed() => { if result.is_err() { break; } }, } } diff --git a/crates/sqlbuild-orchestration/src/webhooks/mod.rs b/crates/sqlbuild-orchestration/src/webhooks/mod.rs new file mode 100644 index 0000000000..eea9ff20d1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/mod.rs @@ -0,0 +1,6 @@ +//! Operator-configured, durable outbound delivery without user-code or scheduler ownership. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/webhooks/models.rs b/crates/sqlbuild-orchestration/src/webhooks/models.rs new file mode 100644 index 0000000000..d71271a5c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/models.rs @@ -0,0 +1,61 @@ +//! A closed event source determines its independent durable acknowledgement namespace. + +use crate::credentials::models::SecurityAuditEvent; +use crate::platform::models::PlatformEventEnvelope; + +#[derive(Clone, Copy, Debug)] +pub enum WebhookSource { + Platform, + SecurityAudit, +} + +impl WebhookSource { + pub(crate) const fn deliveries(self) -> &'static str { + match self { + Self::Platform => "sqb_platform_sink_deliveries", + Self::SecurityAudit => "sqb_security_audit_deliveries", + } + } + + pub(crate) const fn label(self) -> &'static str { + match self { + Self::Platform => "Platform", + Self::SecurityAudit => "Security audit", + } + } +} + +/// Neither errors nor Debug may expose an operator endpoint containing credentials. +#[derive(Clone)] +pub struct Webhook { + pub(crate) client: reqwest::Client, + pub(crate) url: reqwest::Url, + pub(crate) sink_id: String, + pub(crate) source: WebhookSource, +} + +impl std::fmt::Debug for Webhook { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Webhook") + .field("source", &self.source) + .finish_non_exhaustive() + } +} + +/// Preserve the existing source envelope, not a delivery-specific copy of its fields. +#[derive(Debug, serde::Serialize)] +#[serde(untagged)] +pub(crate) enum WebhookEvent { + Platform(PlatformEventEnvelope), + SecurityAudit(Box), +} + +impl WebhookEvent { + pub(crate) fn identity(&self) -> (i64, uuid::Uuid) { + match self { + Self::Platform(event) => (event.storage_id, event.event_id), + Self::SecurityAudit(event) => (event.storage_id, event.event_id), + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 4d35d309f9..b951983f1a 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -1,7 +1,17 @@ //! Real durable audit pages retain clock-independent order, live permissions and atomic writes. +use axum::{ + Json, Router, + extract::State, + http::{HeaderMap, StatusCode}, + routing::post, +}; use chrono::{DateTime, Utc}; use serde_json::{Value, json}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU16, Ordering}, +}; use uuid::Uuid; use crate::authorization::main::{ @@ -16,7 +26,7 @@ use crate::credentials::models::{ AdministratorAuthentication, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, PrincipalKind, RoleAssignmentRequest, }; -use crate::credentials::tests::test_types::AuditTestCase; +use crate::credentials::tests::test_types::{AuditTestCase, AuditWebhookReceiver}; use crate::store::errors::ControlError; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, @@ -27,6 +37,9 @@ use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; +use crate::webhooks::_helpers::delivery::deliver_webhook; +use crate::webhooks::main::connect_webhook::connect_webhook; +use crate::webhooks::models::WebhookSource; #[test] fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_and_atomicity_hold() @@ -124,6 +137,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_security_audit_deliveries", + "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", "DROP INDEX sqb_freshness_source_latest", "DELETE FROM sqb_schema_migrations WHERE name='source_staleness'", "DROP INDEX recall_warehouse_sources", @@ -382,3 +397,197 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ }); } } + +#[test] +fn given_committed_security_facts_when_export_retries_then_only_successful_receipts_advance_commit_order() + { + let test_cases = [ + AuditTestCase { + description: "SQLite durable audit export", + backend: Backend::Sqlite, + expected_page_length: 100, + }, + AuditTestCase { + description: "PostgreSQL durable audit export", + backend: Backend::Postgres, + expected_page_length: 100, + }, + ]; + let scenario: Value = serde_json::from_str( + &std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json", + )) + .expect("delivery fixture"), + ) + .expect("delivery scenario"); + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "principal:orders-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("peer")), + }; + let mut expected = Vec::new(); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("audit action"); + for index in 0..scenario["events"].as_i64().expect("count") { + // Deliberately decreasing event clocks must not change delivery order. + transaction.now = "2026-01-01T00:00:00Z" + .parse::>() + .expect("clock") + - chrono::TimeDelta::seconds(index); + let id = Uuid::now_v7(); + expected.push(json!({"storage_id":index+1,"event_id":id,"occurred_at":transaction.now, + "actor":context.actor,"granting_role":context.granting_role,"request_id":context.request_id, + "ip_address":"127.0.0.1","action":scenario["action"],"scope_json":scenario["scope"], + "before_json":scenario["before"],"after_json":scenario["after"]})); + transaction = append_audit( + transaction, + &context, + (id, scenario["action"].as_str().expect("action")), + ( + scenario["scope"].clone(), + scenario["before"].clone(), + scenario["after"].clone(), + ), + ) + .await + .expect("append"); + } + commit(transaction).await.expect("commit source actions"); + let requests = Arc::new(Mutex::new(Vec::::new())); + let status = Arc::new(AtomicU16::new(503)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener"); + let endpoint = format!( + "http://audit:sample-password@{}/audit?token=sample-token", + listener.local_addr().expect("address") + ); + let app = Router::new() + .route( + "/audit", + post( + |State(AuditWebhookReceiver { requests, status }): State< + AuditWebhookReceiver, + >, + headers: HeaderMap, + Json(event): Json| async move { + assert_eq!( + headers + .get("Idempotency-Key") + .expect("key") + .to_str() + .expect("text"), + event["event_id"].as_str().expect("event id") + ); + requests.lock().expect("requests").push(event); + ( + StatusCode::from_u16(status.load(Ordering::SeqCst)) + .expect("status"), + [(axum::http::header::LOCATION, "/audit")], + ) + }, + ), + ) + .with_state(AuditWebhookReceiver { + requests: requests.clone(), + status: status.clone(), + }); + let (stop, shutdown) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + drop(shutdown.await); + }) + .await + }); + let sink = + connect_webhook(&endpoint, WebhookSource::SecurityAudit).expect("audit sink"); + assert!(!format!("{sink:?}").contains("sample-password")); + assert!(deliver_webhook(store, &sink).await.is_err()); + assert_eq!( + requests.lock().expect("requests").len(), + 1, + "failure stops this page" + ); + status.store(307, Ordering::SeqCst); + assert!( + deliver_webhook(store, &sink).await.is_err(), + "never redirect audit facts" + ); + assert_eq!(requests.lock().expect("requests").len(), 2); + status.store(204, Ordering::SeqCst); + let fault = TransactionFault { + description: "lost security export acknowledgement", + table: "sqb_security_audit_deliveries", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'test acknowledgement failure')", + postgres_body: "RAISE EXCEPTION 'test acknowledgement failure'", + }; + install_fault(store, &fault).await; + assert!(deliver_webhook(store, &sink).await.is_err()); + remove_fault(store, &fault).await; + let replica = second_store(store).await; + let restarted = + connect_webhook(&endpoint, WebhookSource::SecurityAudit).expect("restart"); + assert_eq!( + deliver_webhook(&replica, &restarted) + .await + .expect("first bounded page"), + test_case.expected_page_length, + "{}", + test_case.description + ); + assert_eq!( + deliver_webhook(&replica, &restarted) + .await + .expect("remaining page"), + 5 + ); + assert_eq!( + deliver_webhook(&replica, &restarted) + .await + .expect("durable cursor"), + 0 + ); + // Identical endpoint, independent platform stream: security cursor grants no authority. + let platform = + connect_webhook(&endpoint, WebhookSource::Platform).expect("platform sink"); + assert_eq!( + deliver_webhook(&replica, &platform) + .await + .expect("independent cursor"), + 0 + ); + { + let received = requests.lock().expect("requests"); + assert_eq!(received.len(), expected.len() + 3); + assert_eq!(received[0], expected[0]); + assert_eq!(received[1], expected[0]); + assert_eq!( + received[2], expected[0], + "lost acknowledgement retains full identity" + ); + assert_eq!( + received[3..], + expected, + "every field survives; storage order beats clocks" + ); + } + stop.send(()).expect("stop receiver"); + server + .await + .expect("receiver task") + .expect("receiver stopped"); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index e7d8036e37..a2bf39bdbb 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -1,3 +1,7 @@ +use std::sync::{Arc, Mutex, atomic::AtomicU16}; + +use serde_json::Value; + pub(crate) use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_test_types::Backend; @@ -24,3 +28,9 @@ pub(crate) struct AuditTestCase { pub(crate) backend: Backend, pub(crate) expected_page_length: usize, } + +#[derive(Clone, Debug)] +pub(crate) struct AuditWebhookReceiver { + pub(crate) requests: Arc>>, + pub(crate) status: Arc, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs index d2aeeea1c8..d8ae4b05e0 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs @@ -6,12 +6,12 @@ use axum::Router; use axum::routing::post; use uuid::Uuid; -use crate::platform::_helpers::webhook_delivery::deliver_platform_events; -use crate::platform::main::connect_platform_webhook::connect_platform_webhook; -use crate::platform::main::run_platform_sink::run_platform_sink; use crate::platform::main::run_watchdog::run_watchdog; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; +use crate::webhooks::_helpers::delivery::deliver_webhook; +use crate::webhooks::main::{connect_webhook::connect_webhook, run_webhook::run_webhook}; +use crate::webhooks::models::WebhookSource; use crate::credentials::tests::location_helpers::create_location; use crate::platform::tests::helpers::{ @@ -102,11 +102,12 @@ fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_s }) .await }); - let sink = connect_platform_webhook(&endpoint).expect("configure webhook"); + let sink = + connect_webhook(&endpoint, WebhookSource::Platform).expect("configure webhook"); let diagnostic = format!("{sink:?}"); assert!(!diagnostic.contains("sample-password")); assert!(!diagnostic.contains("sample-token")); - assert!(deliver_platform_events(store, &sink).await.is_err()); + assert!(deliver_webhook(store, &sink).await.is_err()); assert_eq!( state.lock().expect("receiver").requests.len(), 1, @@ -122,15 +123,17 @@ fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_s postgres_body: "RAISE EXCEPTION 'test receipt failure'", }; install_fault(store, &fault).await; - assert!(deliver_platform_events(store, &sink).await.is_err()); + assert!(deliver_webhook(store, &sink).await.is_err()); remove_fault(store, &fault).await; let second = second_store(store).await; - let restarted = connect_platform_webhook(&endpoint).expect("restart webhook client"); + let restarted = connect_webhook(&endpoint, WebhookSource::Platform) + .expect("restart webhook client"); let (sink_stop, sink_shutdown) = tokio::sync::watch::channel(false); let sink_store = second.clone(); - let dispatcher = tokio::spawn(async move { - run_platform_sink(&sink_store, restarted, sink_shutdown).await - }); + let dispatcher = + tokio::spawn( + async move { run_webhook(&sink_store, restarted, sink_shutdown).await }, + ); wait_for_webhook(&receiver, &dispatcher, test_case.expected_events + 2).await; sink_stop.send(true).expect("stop dispatcher"); tokio::time::timeout(Duration::from_secs(2), dispatcher) @@ -140,15 +143,13 @@ fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_s .expect("dispatcher stopped"); // Shutdown may cancel the last receipt after the remote endpoint accepted it. assert!( - deliver_platform_events(store, &sink) + deliver_webhook(store, &sink) .await .expect("reconcile in-flight delivery") <= 1 ); assert_eq!( - deliver_platform_events(store, &sink) - .await - .expect("durable cursor"), + deliver_webhook(store, &sink).await.expect("durable cursor"), 0 ); { diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 5c9433e1c6..c1504c9d5f 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -173,6 +173,8 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .await .expect("pre-sensor predecessor"); for sql in [ + "DROP TABLE sqb_security_audit_deliveries", + "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", "DROP INDEX sqb_freshness_source_latest", "DELETE FROM sqb_schema_migrations WHERE name='source_staleness'", "DELETE FROM recall_events WHERE producer='warehouse' AND event_type='warehouse_source_freshness'", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 11a7038c12..2a3756803c 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_security_audit_deliveries", + "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", "DROP INDEX sqb_freshness_source_latest", "DELETE FROM sqb_schema_migrations WHERE name='source_staleness'", "DROP INDEX recall_warehouse_sources", diff --git a/fensu.toml b/fensu.toml index c614ff7a70..eb37274291 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1578,9 +1578,9 @@ reason = "Browser transport models implement Debug manually to redact passwords [[targets.rust.rule_exceptions]] rule = "FPRSS201" -path = "crates/sqlbuild-orchestration/src/platform/models.rs" -symbols = ["PlatformWebhook"] -reason = "PlatformWebhook implements Debug manually to redact its endpoint, which may contain delivery credentials; other models derive Debug." +path = "crates/sqlbuild-orchestration/src/webhooks/models.rs" +symbols = ["Webhook"] +reason = "Webhook implements Debug manually to redact its endpoint, which may contain delivery credentials; other models derive Debug." [[targets.rust.rule_exceptions]] rule = "FPRSS201" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py index 36d2ea7541..e85570e8ee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -96,6 +96,8 @@ def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeploy "--env", "SQLBUILD_PLATFORM_WEBHOOK_URL=http://127.0.0.1:1/events?token=synthetic-delivery-credential", "--env", + "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL=http://127.0.0.1:1/audit?token=synthetic-audit-credential", + "--env", "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", *mounts, image_id, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 4214a28885..5469c313c8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -517,6 +517,7 @@ def snapshot_api( environment: dict[str, str] = os.environ.copy() environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) + environment.pop("SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL", None) with log.open("wb") as stderr: process: subprocess.Popen[bytes] = subprocess.Popen( [ @@ -633,6 +634,7 @@ def scheduled_worker( "import os\nfrom sqlbuild.orchestration import job, schedule\n" "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" + " assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' @@ -699,6 +701,10 @@ def scheduled_worker( "SQLBUILD_PLATFORM_WEBHOOK_URL", "http://127.0.0.1:1/events?token=synthetic-delivery-credential", ) + monkeypatch.setenv( + "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL", + "http://127.0.0.1:1/audit?token=synthetic-audit-credential", + ) log: Path = directory / "scheduler.log" with location_process( [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py index ba69f29812..5c85fca197 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py @@ -70,6 +70,7 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus pidfile: Path = tmp_path / "interpreter.pid" declaration: str = ( "import os\nfrom pathlib import Path\n" + "assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" f"Path({str(pidfile)!r}).write_text(str(os.getpid()))\n" + Path("benchmarks/duckshop/edge_cases/orchestration/development_orders.py").read_text() ) @@ -94,7 +95,10 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus argument.format(control=control) for argument in test_case.arguments ] environment: dict[str, str] = os.environ | { - "SQLBUILD_CONTROL_DATABASE_URL": "unsupported://not-development" + "SQLBUILD_CONTROL_DATABASE_URL": "unsupported://not-development", + "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL": ( + "http://127.0.0.1:1/audit?token=synthetic-audit-credential" + ), } _ = environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) log: Path = tmp_path / "development.log" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index c858549b22..faeb3c93cb 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -54,7 +54,8 @@ "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " - "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL'):\n" + "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL', " + "'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL'):\n" " assert name not in os.environ, name\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' From cc73cddf3d6724cba6e0f3b1ebfcc8eb8134b901 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:15:01 +0000 Subject: [PATCH 098/138] feat: synchronize authoritative warehouse catalogs --- .../orchestration/warehouse_catalog.json | 29 ++ clients/typescript/src/schema.d.ts | 95 ++++++- clients/typescript/tests/types.ts | 19 ++ .../src/native_commands/_helpers/execution.rs | 18 +- .../main/_location_metadata.rs | 4 +- .../src/native_commands/models.rs | 4 + .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 23 +- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 13 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/main/migrate_store.rs | 5 +- .../src/store/warehouse_catalog.sql | 12 + .../warehouse_metadata/_helpers/catalog.rs | 70 +++++ .../src/warehouse_metadata/_helpers/mod.rs | 1 + .../_helpers/reconciliation.rs | 23 +- .../src/warehouse_metadata/constants.rs | 1 + .../warehouse_metadata/main/ingest_catalog.rs | 84 ++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../warehouse_metadata/main/read_watermark.rs | 7 +- .../src/warehouse_metadata/models.rs | 47 +++- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_warehouse_metadata.rs | 7 + .../api/warehouse_metadata/catalog/helpers.rs | 224 +++++++++++++++ .../tests/credentials/test_audit.rs | 2 + .../tests/sensors/test_sources.rs | 2 + .../tests/store/helpers.rs | 2 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../_helpers/native_host/warehouse_catalog.rs | 132 +++++++++ .../native_host/warehouse_metadata.rs | 15 + .../publication/_test_types.py | 15 + .../publication/catalog/helpers.py | 37 +++ .../publication/test_warehouse_catalog.py | 259 ++++++++++++++++++ 33 files changed, 1141 insertions(+), 19 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_catalog.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json new file mode 100644 index 0000000000..524f87cdc6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json @@ -0,0 +1,29 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "name": "orders_🧾", + "relation_type": "BASE TABLE", + "columns": [ + {"name": "order_id", "data_type": "INTEGER"}, + {"name": "note\"text", "data_type": "VARCHAR"} + ] + }, + "create_sql": "CREATE TABLE \"orders_🧾\" (order_id INTEGER, \"note\"\"text\" VARCHAR)", + "alter_sql": "ALTER TABLE \"orders_🧾\" ADD COLUMN shipped INTEGER", + "replace_sql": ["DROP TABLE \"orders_🧾\"", "CREATE VIEW \"orders_🧾\" AS SELECT 7 AS order_id"], + "drop_sql": "DROP VIEW \"orders_🧾\"", + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "extra_relations": 104, + "non_authoritative_database": null, + "postgres_sql": [ + "CREATE SCHEMA main", + "CREATE TABLE main.orders (order_id INTEGER, amount NUMERIC)", + "CREATE TABLE main.\"Orders\" (note TEXT, shipped BOOLEAN)" + ], + "postgres_records": [ + {"state_database": null, "state_schema": "main", "name": "Orders", "relation_type": "BASE TABLE", "columns": [{"name": "note", "data_type": "text"}, {"name": "shipped", "data_type": "boolean"}]}, + {"state_database": null, "state_schema": "main", "name": "orders", "relation_type": "BASE TABLE", "columns": [{"name": "order_id", "data_type": "integer"}, {"name": "amount", "data_type": "numeric"}]} + ], + "lifecycle": ["create", "unchanged scan", "column added", "table becomes view", "drop", "empty mirror rebuild"] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 040acca1e8..43b8e7b9ad 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -291,6 +291,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/catalog": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_catalog"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/fingerprints": { parameters: { query?: never; @@ -694,6 +710,14 @@ export interface components { /** Format: uuid */ run_id: string; }; + CatalogBatch: { + /** Format: int64 */ + generation: number; + reconciliation: components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseCatalogRecord"][]; + /** Format: uuid */ + session_id: string; + }; CatchUp: "latest" | { /** @description Process the oldest bounded batch; later due times remain due for the next transaction. */ all: { @@ -1497,6 +1521,18 @@ export interface components { /** Format: int64 */ violation_count?: number | null; }; + WarehouseCatalogColumn: { + data_type: string; + name: string; + }; + /** @description Physical identity and ordered columns come from the adapter catalog, not fingerprints. */ + WarehouseCatalogRecord: { + columns: components["schemas"]["WarehouseCatalogColumn"][]; + name: string; + relation_type: string; + state_database?: string | null; + state_schema: string; + }; /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ WarehouseFingerprintRecord: { definition: string; @@ -1534,10 +1570,10 @@ export interface components { value_kind: string; }; /** - * @description Only immutable history tables support a timestamp watermark. + * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -2425,6 +2461,61 @@ export interface operations { }; }; }; + report_catalog: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CatalogBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_fingerprints: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 6b2df6e31d..5b67c410b3 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -67,6 +67,25 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/microba params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const catalogHistory: components["schemas"]["WarehouseHistoryKind"] = "catalog"; +const catalogColumn: components["schemas"]["WarehouseCatalogColumn"] = { + name: "order_id", data_type: "INTEGER", +}; +void catalogHistory; +void catalogColumn; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/catalog", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { + session_id: "synthetic-session", generation: 1, records: [], + reconciliation: { + scan_id: "synthetic-scan", page: 0, state_database: null, state_schema: "main", + day: null, complete: true, after: null, + }, + }, +}); +// @ts-expect-error Catalog scans must include reconciliation; incremental imports cannot remove relations safely. +const incompleteCatalog: components["schemas"]["CatalogBatch"] = { session_id: "synthetic-session", generation: 1, records: [] }; +void incompleteCatalog; const sourceCondition: components["schemas"]["SensorCondition"] = { kind: "source_new_data", selection: ["source:raw_orders"], }; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 9f3bb587ff..5f5fe919a8 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -350,9 +350,10 @@ pub(crate) fn history_request(state: &LocationMetadataSync) -> Result, C )])); } let kind = state.kind().as_str(); - serde_json::to_vec(&json!({(kind): { + let request = json!({(kind): { "location":state.location(),"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit - }})).map_err(|error| ControlError::Invalid(error.to_string())) + }}); + serde_json::to_vec(&request).map_err(|error| ControlError::Invalid(error.to_string())) } /// Reserve the largest mirror cursor before freezing: receipts prohibit later content splitting. @@ -366,7 +367,7 @@ pub(crate) fn freeze_history_page( "reconciliation":{"scan_id":state.scan_id,"page":state.page,"state_database":location.database, "state_schema":location.schema,"day":null,"complete":page.next_after.is_none(), "after":WarehouseReconciliationCursor { observed_at: DateTime::::MAX_UTC, event_id: Uuid::nil() }}}); - if !state.reconcile { + if !state.reconciling() { body["reconciliation"] = serde_json::Value::Null; } if serde_json::to_vec(&body) @@ -384,7 +385,7 @@ pub(crate) fn freeze_history_page( )])); } } else { - if state.reconcile { + if state.reconciling() { body["reconciliation"]["after"] = serde_json::Value::Null; } state.pending = Some(LocationMetadataPage { @@ -401,6 +402,15 @@ pub(crate) async fn deliver_metadata_page( kind: WarehouseHistoryKind, ) -> Result { match kind { + WarehouseHistoryKind::Catalog => { + client + .sync_catalog( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid catalog batch"))?, + ) + .await + } WarehouseHistoryKind::Fingerprints => { client .sync_fingerprints( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index f93388caeb..e93c96caf2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -10,7 +10,7 @@ use sqlbuild_orchestration::platform::models::{ }; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryPage, WarehouseWatermarkRead, + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseWatermarkRead, }; use tokio::time::Instant; @@ -46,7 +46,7 @@ pub(super) async fn sync_location_metadata( if state.index == 0 { state.reconcile = Instant::now() >= state.reconcile_at; } - if !state.reconcile { + if !state.reconcile && state.kind() != WarehouseHistoryKind::Catalog { let request = WarehouseWatermarkRead { session_id: poll.session_id, generation: state.generation, diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index cd12aba21e..b609c33f17 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -204,6 +204,10 @@ pub(crate) struct LocationMetadataSync { } impl LocationMetadataSync { + pub(crate) fn reconciling(&self) -> bool { + self.reconcile || self.kind() == WarehouseHistoryKind::Catalog + } + pub(crate) fn kind(&self) -> WarehouseHistoryKind { WarehouseHistoryKind::ALL[self.index % WarehouseHistoryKind::ALL.len()] } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index bb7cda21f7..23517df972 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -59,6 +59,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_catalog(&store, (location_id, &target), &batch).await?, + )) +} + #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/watermark", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = WarehouseWatermarkRead, responses((status = 200, body = Option>), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 1a623f0e0d..28b74c5011 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -55,6 +55,7 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report_migrations, warehouse_metadata::report_old_name_views, warehouse_metadata::report_microbatches, + warehouse_metadata::report_catalog, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index e4b218c28e..f83364e585 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,11 +10,20 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - AuditResultsBatch, FingerprintBatch, JanitorEventsBatch, MicrobatchesBatch, MigrationsBatch, - OldNameViewsBatch, SourceFreshnessBatch, WarehouseSyncResponse, WarehouseWatermarkRead, + AuditResultsBatch, CatalogBatch, FingerprintBatch, JanitorEventsBatch, MicrobatchesBatch, + MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, WarehouseSyncResponse, + WarehouseWatermarkRead, }; impl LocationWorkClient { + pub async fn sync_catalog( + &self, + target: &str, + batch: &CatalogBatch, + ) -> Result { + self.sync_metadata((target, "catalog"), batch).await + } + pub async fn report_health(&self, report: &LocationHealthReport) -> Result { let endpoint = self.connection.endpoint(&[ "api", diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 3d3bff65c6..4c8fa57cc8 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -34,6 +34,7 @@ pub(crate) const WAREHOUSE_MICROBATCHES: &str = include_str!("../warehouse_micro pub(crate) const SOURCE_SENSORS: &str = include_str!("../source_sensors.sql"); pub(crate) const SOURCE_STALENESS: &str = include_str!("../source_staleness.sql"); pub(crate) const SECURITY_AUDIT_SINK: &str = include_str!("../security_audit_sink.sql"); +pub(crate) const WAREHOUSE_CATALOG: &str = include_str!("../warehouse_catalog.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 260219a98e..e76623dec5 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -8,8 +8,8 @@ use crate::store::_helpers::migration::{ DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SECURITY_AUDIT_SINK, SENSORS, SOURCE_SENSORS, - SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, - WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, + SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_CATALOG, WAREHOUSE_FINGERPRINTS, + WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; @@ -61,6 +61,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("source_sensors", SOURCE_SENSORS), ("source_staleness", SOURCE_STALENESS), ("security_audit_sink", SECURITY_AUDIT_SINK), + ("warehouse_catalog", WAREHOUSE_CATALOG), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_catalog.sql b/crates/sqlbuild-orchestration/src/store/warehouse_catalog.sql new file mode 100644 index 0000000000..8439c7558a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_catalog.sql @@ -0,0 +1,12 @@ +CREATE TABLE sqb_warehouse_catalog ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_warehouse_catalog_namespace ON sqb_warehouse_catalog + (location_id, target, state_schema, state_database, observed_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs new file mode 100644 index 0000000000..6fe74655d9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs @@ -0,0 +1,70 @@ +//! Structural facts and their current projection always commit together. + +use std::collections::BTreeSet; + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::CatalogBatch; + +pub(crate) fn validate_catalog(batch: &CatalogBatch) -> Result<(), ControlError> { + let page = &batch.reconciliation; + let mut names: BTreeSet<&String> = BTreeSet::new(); + validate_fields( + ("catalog", batch.generation, batch.records.len()), + batch.records.iter().any(|record| { + record.name.trim().is_empty() + || record.relation_type.trim().is_empty() + || !names.insert(&record.name) + }), + page.day.is_some() + || batch.records.iter().any(|record| { + record.state_database != page.state_database + || record.state_schema != page.state_schema + }), + )?; + for record in &batch.records { + if record + .columns + .iter() + .any(|column| column.name.trim().is_empty() || column.data_type.trim().is_empty()) + { + return Err(ControlError::Invalid( + "catalog columns need nonempty names and types".into(), + )); + } + } + Ok(()) +} + +pub(crate) async fn append_catalog_fact( + transaction: ControlTransaction, + (scan_id, page, relation_id): (Uuid, u32, Uuid), + (location_id, target): (Uuid, &str), + (record, present): (Value, bool), +) -> Result { + let event_id = Uuid::new_v5( + &scan_id, + json!((page, relation_id, present)).to_string().as_bytes(), + ); + let occurred_at = transaction.now; + let (transaction, outcome) = append_control_event(transaction, ControlEvent { + event_id, + invocation_id: format!("warehouse/catalog/{relation_id}"), + event_type: "warehouse_catalog_changed", + producer: "warehouse", + occurred_at, + run_id: None, + payload: json!({"location_id":location_id,"target":target,"relation_id":relation_id,"record":record,"present":present}), + }).await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "catalog observation conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs index 9849049bc6..08f156dd76 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod catalog; pub(crate) mod events; pub(crate) mod fingerprints; pub(crate) mod freshness; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs index a3c2265bfc..3130a1f69d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/reconciliation.rs @@ -8,7 +8,8 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::_helpers::catalog::append_catalog_fact; +use crate::warehouse_metadata::constants::{CATALOG_MIRROR, MAX_HISTORY_RECORDS}; use crate::warehouse_metadata::models::{ WarehouseReconciliationCursor, WarehouseReconciliationPage, WarehouseSyncResponse, }; @@ -176,11 +177,29 @@ pub(crate) async fn finish_reconciliation( .map(|index| format!("${index}")) .collect::>() .join(","); + let returning = if table == CATALOG_MIRROR { + "event_id,record_json" + } else { + "event_id" + }; let removed; (transaction, removed) = query(transaction, &format!( "DELETE FROM {table} WHERE event_id IN ({placeholders}) \ AND received_at<(SELECT received_at FROM sqb_warehouse_scan_pages WHERE scan_id=$1 AND page=0) \ - AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_scan_keys k WHERE k.scan_id=$1 AND k.event_id={table}.event_id) RETURNING event_id"), parameters).await?; + AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_scan_keys k WHERE k.scan_id=$1 AND k.event_id={table}.event_id) RETURNING {returning}"), parameters).await?; + if table == CATALOG_MIRROR { + for row in &removed { + let relation_id = serde_json::from_value(row["event_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction = append_catalog_fact( + transaction, + (page.scan_id, page.page, relation_id), + (location_id, target), + (row["record_json"].clone(), false), + ) + .await?; + } + } response.applied = !removed.is_empty(); Ok((transaction, response)) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 2615174d34..7770ce5e30 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -1,3 +1,4 @@ //! Limits bound each authenticated warehouse observation transaction. pub const MAX_HISTORY_RECORDS: usize = 100; +pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs new file mode 100644 index 0000000000..5a50d6c93e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs @@ -0,0 +1,84 @@ +//! A full catalog scan records structural changes, never inferred relation existence. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::catalog::{append_catalog_fact, validate_catalog}; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, scoped_event_id, +}; +use crate::warehouse_metadata::_helpers::reconciliation::begin_reconciliation; +use crate::warehouse_metadata::models::{CatalogBatch, WarehouseSyncResponse}; + +pub async fn ingest_catalog( + store: &ControlStore, + scope: (Uuid, &str), + batch: &CatalogBatch, +) -> Result { + validate_catalog(batch)?; + let page = &batch.reconciliation; + let transaction = begin_ingestion(store, scope, (batch.session_id, batch.generation)).await?; + let (mut transaction, received) = begin_reconciliation( + transaction, + ("sqb_warehouse_catalog", scope.0, scope.1), + page, + json!(batch), + ) + .await?; + let mut keys: Vec = Vec::new(); + let mut applied = false; + if !received { + for record in &batch.records { + let relation_id = scoped_event_id( + scope, + ( + record.state_database.as_deref(), + &record.state_schema, + "catalog", + ), + &record.name, + )?; + keys.push(relation_id); + let existing; + (transaction, existing) = query( + transaction, + "SELECT record_json FROM sqb_warehouse_catalog WHERE event_id=$1", + vec![Parameter::Id(relation_id)], + ) + .await?; + let record_json = json!(record); + if existing + .first() + .is_some_and(|row| row["record_json"] == record_json) + { + continue; + } + transaction = append_catalog_fact( + transaction, + (page.scan_id, page.page, relation_id), + scope, + (record_json.clone(), true), + ) + .await?; + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_warehouse_catalog (event_id,location_id,target,state_database,state_schema,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$6) ON CONFLICT (event_id) DO UPDATE SET \ + observed_at=excluded.observed_at,record_json=excluded.record_json,received_at=excluded.received_at", + vec![Parameter::Id(relation_id), Parameter::Id(scope.0), Parameter::Text(scope.1.into()), + Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Time(now), Parameter::Json(record_json)]).await?; + applied = true; + } + } + commit_ingestion( + transaction, + ("sqb_warehouse_catalog", scope.0, scope.1), + (Some(page), &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 1eefd96bc0..6435a58598 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod bootstrap_source_observations; pub mod ingest_audit_results; +pub mod ingest_catalog; pub mod ingest_fingerprints; pub mod ingest_janitor_events; pub mod ingest_microbatches; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs index 6f0420bbdd..35ec55edfd 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs @@ -7,7 +7,7 @@ use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; use crate::store::models::Parameter; use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; -use crate::warehouse_metadata::models::WarehouseWatermarkRead; +use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseWatermarkRead}; pub async fn read_watermark( store: &crate::store::models::ControlStore, @@ -16,6 +16,11 @@ pub async fn read_watermark( ) -> Result>, ControlError> { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + if request.kind == WarehouseHistoryKind::Catalog { + return Err(ControlError::Invalid( + "catalog observations require full scans, not timestamp watermarks".into(), + )); + } let table = request.kind.mirror_table(); let mut parameters = vec![ Parameter::Id(scope.0), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index cc04480bf3..ffed8ec562 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -278,7 +278,7 @@ pub struct WarehouseStateLocation { pub schema: String, } -/// Only immutable history tables support a timestamp watermark. +/// Catalog observations require full scans; only immutable history supports a watermark. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum WarehouseHistoryKind { @@ -289,10 +289,11 @@ pub enum WarehouseHistoryKind { Migrations, OldNameViews, Microbatches, + Catalog, } impl WarehouseHistoryKind { - pub const ALL: [Self; 7] = [ + pub const ALL: [Self; 8] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -300,6 +301,7 @@ impl WarehouseHistoryKind { Self::Migrations, Self::OldNameViews, Self::Microbatches, + Self::Catalog, ]; #[must_use] @@ -312,6 +314,7 @@ impl WarehouseHistoryKind { Self::Migrations => "migrations", Self::OldNameViews => "old_name_views", Self::Microbatches => "microbatches", + Self::Catalog => "catalog", } } @@ -325,6 +328,7 @@ impl WarehouseHistoryKind { Self::Migrations => "sqb_warehouse_migrations", Self::OldNameViews => "sqb_warehouse_old_name_views", Self::Microbatches => "sqb_warehouse_microbatches", + Self::Catalog => "sqb_warehouse_catalog", } } } @@ -440,6 +444,7 @@ pub enum WarehouseHistoryRead { Migrations(StateEventsRead), OldNameViews(StateEventsRead), Microbatches(StateEventsRead), + Catalog(CatalogRead), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -452,6 +457,7 @@ pub enum WarehouseHistoryRecord { Migration(Box), OldNameView(Box), Microbatch(Box), + Catalog(WarehouseCatalogRecord), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -481,3 +487,40 @@ pub(crate) struct StateEventsImport<'a> { pub reconciliation: Option<&'a WarehouseReconciliationPage>, pub payload: serde_json::Value, } + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseCatalogColumn { + pub name: String, + pub data_type: String, +} + +/// Physical identity and ordered columns come from the adapter catalog, not fingerprints. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseCatalogRecord { + pub state_database: Option, + pub state_schema: String, + pub name: String, + pub relation_type: String, + pub columns: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct CatalogBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + pub reconciliation: WarehouseReconciliationPage, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CatalogRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 1]>, + /// Catalog reads reject watermarks; the shared reader envelope supplies null. + pub observed_since: Option>, + pub limit: Option, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 5a14957b9e..2bead6b842 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -38,6 +38,8 @@ mod test_types; mod test_warehouse_metadata; #[path = "test_worker.rs"] mod test_worker; +#[path = "warehouse_metadata/catalog/helpers.rs"] +mod warehouse_catalog_helpers; #[path = "warehouse_metadata/events/helpers.rs"] mod warehouse_events_helpers; #[path = "warehouse_metadata/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 6fda83aec2..beb5f9c29c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -357,6 +357,13 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit ) .await; } + crate::api::tests::warehouse_catalog_helpers::catalog_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs new file mode 100644 index 0000000000..fad9b3cffc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs @@ -0,0 +1,224 @@ +//! Authoritative catalog changes, atomic facts and bounded removal through real HTTP. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::LocationWorkClient; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::models::CatalogBatch; + +pub(crate) async fn catalog_history( + store: &ControlStore, + outbound: &LocationWorkClient, + (session_id, preview_session): (Uuid, Uuid), + (client, freshness_url, bearer): (&Client, &str, &str), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json"); + let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("catalog fixture"); + let record = fixture["record"].clone(); + let url = freshness_url.replace("source-freshness", "catalog"); + let mut batch = json!({"session_id":session_id,"generation":1,"records":[record], + "reconciliation":{"scan_id":Uuid::now_v7(),"page":0,"state_database":null,"state_schema":"main","day":null,"complete":true,"after":null}}); + let typed: CatalogBatch = serde_json::from_value(batch.clone()).expect("catalog batch"); + assert!( + outbound + .sync_catalog("development", &typed) + .await + .expect("native catalog client") + .applied + ); + assert!( + !outbound + .sync_catalog("development", &typed) + .await + .expect("lost response retry") + .applied + ); + let initial = evidence(store).await; + assert_eq!(initial["facts"].as_array().expect("facts").len(), 1); + for (field, value, status) in [ + ("generation", json!(2), StatusCode::CONFLICT), + ( + "session_id", + json!(preview_session), + StatusCode::BAD_REQUEST, + ), + ("records", json!([record, record]), StatusCode::BAD_REQUEST), + ] { + let mut invalid = batch.clone(); + invalid[field] = value; + post((client, &url, bearer), &invalid, status).await; + } + post( + ( + client, + &url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string(), + ), + bearer, + ), + &batch, + StatusCode::FORBIDDEN, + ) + .await; + batch["reconciliation"]["scan_id"] = json!(Uuid::now_v7()); + assert_eq!( + post((client, &url, bearer), &batch, StatusCode::OK).await["applied"], + false + ); + assert_eq!( + evidence(store).await, + initial, + "unchanged scans do not append structural facts" + ); + batch["reconciliation"]["scan_id"] = json!(Uuid::now_v7()); + batch["records"][0]["columns"][1]["data_type"] = json!("INTEGER"); + for (table, operation) in [ + ("recall_events", "INSERT"), + ("sqb_warehouse_catalog", "UPDATE"), + ("sqb_warehouse_scan_keys", "INSERT"), + ] { + let fault = TransactionFault { + description: "catalog change rollback", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected catalog failure')", + postgres_body: "RAISE EXCEPTION 'injected catalog failure'", + }; + install_fault(store, &fault).await; + post( + (client, &url, bearer), + &batch, + StatusCode::INTERNAL_SERVER_ERROR, + ) + .await; + remove_fault(store, &fault).await; + assert_eq!( + evidence(store).await, + initial, + "fact, projection and receipt roll back together" + ); + } + assert_eq!( + post((client, &url, bearer), &batch, StatusCode::OK).await["applied"], + true + ); + let changed = evidence(store).await; + assert_eq!(changed["facts"].as_array().expect("facts").len(), 2); + assert_eq!( + changed["current"][0]["record_json"]["columns"][1]["data_type"], + "INTEGER" + ); + let mut obsolete: Vec = Vec::new(); + for index in 0..205 { + let mut item = record.clone(); + item["name"] = json!(format!("obsolete-{index:03}")); + obsolete.push(item); + } + for records in obsolete.chunks(100) { + batch["reconciliation"]["scan_id"] = json!(Uuid::now_v7()); + batch["reconciliation"]["complete"] = json!(false); + batch["records"] = json!(records); + post((client, &url, bearer), &batch, StatusCode::OK).await; + } + let before_deletion = evidence(store).await; + batch["reconciliation"]["scan_id"] = json!(Uuid::now_v7()); + batch["records"] = json!([]); + post((client, &url, bearer), &batch, StatusCode::OK).await; + assert_eq!( + evidence(store).await, + before_deletion, + "partial catalog scans cannot remove objects" + ); + batch["reconciliation"]["page"] = json!(1); + batch["reconciliation"]["complete"] = json!(true); + let fault = TransactionFault { + description: "catalog removal fact rollback", + table: "recall_events", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected catalog removal failure')", + postgres_body: "RAISE EXCEPTION 'injected catalog removal failure'", + }; + install_fault(store, &fault).await; + post( + (client, &url, bearer), + &batch, + StatusCode::INTERNAL_SERVER_ERROR, + ) + .await; + remove_fault(store, &fault).await; + assert_eq!( + evidence(store).await, + before_deletion, + "removal without its fact cannot commit" + ); + for index in 0..3 { + let before = evidence(store).await["current"] + .as_array() + .expect("current") + .len(); + let reply = tokio::time::timeout( + std::time::Duration::from_secs(5), + post((client, &url, bearer), &batch, StatusCode::OK), + ) + .await + .expect("bounded catalog reconciliation"); + let after = evidence(store).await["current"] + .as_array() + .expect("current") + .len(); + assert!(before - after <= 100); + batch["reconciliation"]["after"] = reply["next_reconciliation_after"].clone(); + assert_eq!(batch["reconciliation"]["after"].is_null(), index == 2); + } + let removed = evidence(store).await; + assert_eq!(removed["current"], json!([])); + assert_eq!(removed["facts"].as_array().expect("facts").len(), 413); + post((client, &url, bearer), &batch, StatusCode::OK).await; + assert_eq!( + evidence(store).await, + removed, + "committed final-page replay appends no facts" + ); +} + +async fn post( + (client, url, bearer): (&Client, &str, &str), + body: &Value, + expected: StatusCode, +) -> Value { + let response = client + .post(url) + .bearer_auth(bearer) + .json(body) + .send() + .await + .expect("catalog request"); + let status = response.status(); + let body: Value = response.json().await.expect("catalog reply"); + assert_eq!(status, expected, "{body}"); + body +} + +async fn evidence(store: &ControlStore) -> Value { + let (transaction, current) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT event_id,record_json FROM sqb_warehouse_catalog ORDER BY event_id", + vec![], + ) + .await + .expect("projection"); + let (transaction,facts) = query(transaction, + "SELECT event_id,payload,invocation_sequence FROM recall_events WHERE event_type='warehouse_catalog_changed' ORDER BY storage_id",vec![]).await.expect("facts"); + commit(transaction).await.expect("close read"); + json!({"current":current,"facts":facts}) +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index b951983f1a..6c8a9161d5 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -137,6 +137,8 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", "DROP INDEX sqb_freshness_source_latest", diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index c1504c9d5f..0817e93771 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -173,6 +173,8 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .await .expect("pre-sensor predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", "DROP INDEX sqb_freshness_source_latest", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 2a3756803c..a3a067fe17 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,8 @@ pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { .await .expect("predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", "DROP INDEX sqb_freshness_source_latest", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 248bf8cc95..858bd23966 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -14,4 +14,5 @@ pub(crate) mod results; pub(crate) mod sqlite_connection; pub(crate) mod sqlite_cursor; pub(crate) mod state_column_evolution; +pub(crate) mod warehouse_catalog; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs new file mode 100644 index 0000000000..112caea0a2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs @@ -0,0 +1,132 @@ +//! Bounded catalog pages use adapter inspection, never model declarations or warehouse writes. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + CatalogRead, WarehouseCatalogColumn, WarehouseCatalogRecord, WarehouseHistoryPage, + WarehouseHistoryRecord, +}; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::records::fields::field; + +pub(crate) fn read_catalog( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &CatalogRead, +) -> PyResult { + let limit = request.limit.unwrap_or(MAX_HISTORY_RECORDS); + if request.observed_since.is_some() { + return Err(PyValueError::new_err( + "catalog observations require full scans, not timestamp watermarks", + )); + } + if limit == 0 || limit > MAX_HISTORY_RECORDS { + return Err(PyValueError::new_err( + "metadata page limit is outside the supported bounds", + )); + } + let namespace = [ + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schema", + request.location.schema.clone().into_bound_py_any(py)?, + ), + ]; + if !call_with( + &adapter.getattr("lists_relations_authoritatively")?, + &namespace, + )? + .extract::()? + { + return Err(PyValueError::new_err( + "catalog listing is not authoritative for this namespace; scan incomplete", + )); + } + let relations = call_with( + &adapter.getattr("list_relations")?, + &[ + ("connection", connection.clone()), + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schemas", + (request.location.schema.clone(),).into_bound_py_any(py)?, + ), + ], + )?; + let mut selected: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); + for relation in relations.try_iter()? { + let relation = relation?; + let name: String = field(&relation, "name")?.extract()?; + if request + .after + .as_ref() + .and_then(|cursor| cursor[0].as_deref()) + .is_none_or(|after| name.as_str() > after) + { + selected.push((name, relation)); + } + } + selected.sort_by(|left, right| left.0.cmp(&right.0)); + let more = selected.len() > limit; + selected.truncate(limit); + if selected.is_empty() { + return Ok(WarehouseHistoryPage { + records: Vec::new(), + next_after: None, + }); + } + let names = PyTuple::new(py, selected.iter().map(|(name, _)| name))?; + let columns = call_with( + &adapter.getattr("get_all_columns")?, + &[ + ("connection", connection.clone()), + ("database", namespace[0].1.clone()), + ( + "schemas", + (request.location.schema.clone(),).into_bound_py_any(py)?, + ), + ("names", names.into_any()), + ], + )?; + let columns = columns.cast::()?; + let next_after = if more { + selected.last().map(|(name, _)| vec![Some(name.clone())]) + } else { + None + }; + let mut records: Vec = Vec::new(); + for (name, relation) in selected { + let relation_columns = columns.get_item(name.as_str())?.ok_or_else(|| { + PyValueError::new_err("catalog relation columns are unavailable; scan incomplete") + })?; + let mut decoded: Vec = Vec::new(); + for column in relation_columns.try_iter()? { + let column = column?; + decoded.push(WarehouseCatalogColumn { + name: field(&column, "name")?.extract()?, + data_type: field(&column, "type")?.extract()?, + }); + } + records.push(WarehouseHistoryRecord::Catalog(WarehouseCatalogRecord { + state_database: request.location.database.clone(), + state_schema: request.location.schema.clone(), + name, + relation_type: field(&relation, "relation_type")?.extract()?, + columns: decoded, + })); + } + Ok(WarehouseHistoryPage { + records, + next_after, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index cb6d20fafc..9586f0df14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -28,6 +28,7 @@ use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; +use crate::bindings::_helpers::native_host::warehouse_catalog::read_catalog; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; @@ -40,6 +41,7 @@ pub(crate) fn read_history( request: &WarehouseHistoryRead, ) -> PyResult { let location = match request { + WarehouseHistoryRead::Catalog(read) => &read.location, WarehouseHistoryRead::SourceFreshness(read) => &read.location, WarehouseHistoryRead::Fingerprints(read) => &read.location, WarehouseHistoryRead::AuditResults(read) @@ -123,6 +125,9 @@ fn read_page( }; let event_columns: Vec<&str> = event_columns.iter().map(String::as_str).collect(); let (location, after, since, limit, table, columns, timestamp) = match request { + WarehouseHistoryRead::Catalog(read) => { + return read_catalog(py, (adapter, connection), read); + } WarehouseHistoryRead::SourceFreshness(read) => ( &read.location, read.after.as_ref().map(|cursor| cursor.as_slice()), @@ -367,6 +372,11 @@ fn read_page( }; } let observed = match &record { + WarehouseHistoryRecord::Catalog(_) => { + return Err(PyValueError::new_err( + "catalog does not use timestamp history filtering", + )); + } WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, WarehouseHistoryRecord::Fingerprint(record) => record.ts, WarehouseHistoryRecord::AuditResult(record) => record.occurred_at, @@ -395,6 +405,11 @@ fn decode_record( (columns, timestamp): (&[&str], &str), ) -> PyResult { let (location, record) = match request { + WarehouseHistoryRead::Catalog(_) => { + return Err(PyValueError::new_err( + "catalog does not use state-table row decoding", + )); + } WarehouseHistoryRead::SourceFreshness(read) => (&read.location, row_record(row)?), WarehouseHistoryRead::Fingerprints(read) => ( &read.location, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index d8b325146a..56ccde09be 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -228,3 +228,18 @@ class InstalledRecoveryTestCase: description: str executable: str expected_rejection: str + + +@dataclass(frozen=True) +class CatalogInspectionTestCase: + description: str + adapter_name: str + connection_key: str + expected_table_type: str + expected_view_type: str + + +@dataclass(frozen=True) +class CatalogBoundaryTestCase: + description: str + expected_page_delivered: bool diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py new file mode 100644 index 0000000000..51d7aa990e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -0,0 +1,37 @@ +"""Observe the installed location's catalog projection and original structural facts.""" + +import json +import sqlite3 +import time +from contextlib import closing +from pathlib import Path +from subprocess import Popen +from typing import Any + + +def wait_for_catalog( + control: Path, expected: list[dict[str, Any]], process: Popen[bytes], log: Path +) -> None: + """Wait for the exact synthetic relation, not a count that unrelated rows can satisfy.""" + deadline: float = time.monotonic() + 45 + observed: list[dict[str, Any]] = [{}] + while observed != expected: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (observed, expected, log.read_text()) + with closing(sqlite3.connect(control)) as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT record_json FROM sqb_warehouse_catalog " + "WHERE json_extract(record_json, '$.name')='orders_🧾'" + ).fetchall() + observed = [json.loads(row[0]) for row in rows] + time.sleep(0.05) + + +def catalog_facts(control: Path) -> list[dict[str, Any]]: + """Return commit-ordered facts for the exact Unicode relation.""" + with closing(sqlite3.connect(control)) as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT payload FROM recall_events WHERE event_type='warehouse_catalog_changed' " + "AND json_extract(payload, '$.record.name')='orders_🧾' ORDER BY storage_id" + ).fetchall() + return [json.loads(row[0]) for row in rows] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py new file mode 100644 index 0000000000..1ca69c726f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py @@ -0,0 +1,259 @@ +"""Real adapter catalog inspection and automatic standalone location synchronization.""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +import time +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CatalogBoundaryTestCase, + CatalogInspectionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + catalog_facts, + wait_for_catalog, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + open_legacy_warehouse, + snapshot_api, + wait_for_generation, + warehouse_history_page, + warehouse_history_pages, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogInspectionTestCase("DuckDB catalog", "duckdb", "database", "BASE TABLE", "VIEW"), + CatalogInspectionTestCase("SQLite catalog", "sqlite", "path", "BASE TABLE", "VIEW"), + ], + ids=lambda case: case.description, +) +def test_given_real_catalog_when_reading_pages_then_columns_and_physical_types_are_read_without_mutation( + test_case: CatalogInspectionTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + for index in range(fixture["extra_relations"]): + _ = adapter.execute( + connection=connection, sql=f"CREATE TABLE catalog_orders_{index:03} (amount INTEGER)" + ) + adapter.close(connection) + before: str = hashlib.sha256((project / "warehouse.db").read_bytes()).hexdigest() + pages: list[dict[str, Any]] = warehouse_history_pages(project, "catalog") + assert tuple(len(page["records"]) for page in pages) == (100, 7) + records: list[dict[str, Any]] = [] + for page in pages: + records.extend(page["records"]) + assert [record["name"] for record in records] == sorted(record["name"] for record in records) + by_name: dict[str, Any] = {record["name"]: record for record in records} + expected: dict[str, Any] = fixture["record"] | {"relation_type": test_case.expected_table_type} + assert by_name[expected["name"]] == expected + assert by_name[expected["name"]]["relation_type"] == test_case.expected_table_type + assert hashlib.sha256((project / "warehouse.db").read_bytes()).hexdigest() == before + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [CatalogBoundaryTestCase("Case-distinct physical identities", True)], + ids=lambda case: case.description, +) +def test_given_case_distinct_postgres_relations_when_reading_catalog_then_columns_remain_exact( + test_case: CatalogBoundaryTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "postgres"\n' + '[connections.local]\nconninfo = "${ENV:SQLBUILD_CATALOG_TEST_URL}"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + }, + ) + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + url: str = postgres.get_connection_url() + monkeypatch.setenv("SQLBUILD_CATALOG_TEST_URL", url) + adapter: PostgresAdapter = PostgresAdapter() + connection: Any = adapter.connect({"conninfo": url}) + for sql in fixture["postgres_sql"]: + _ = adapter.execute(connection=connection, sql=sql) + adapter.close(connection) + page: Any = warehouse_history_page(project, "catalog") + assert (page is not None) is test_case.expected_page_delivered + assert page == {"records": fixture["postgres_records"], "next_after": None} + + +@pytest.mark.parametrize( + "test_case", + [CatalogBoundaryTestCase("Non-authoritative schema-only namespace", False)], + ids=lambda case: case.description, +) +def test_given_non_authoritative_adapter_scope_when_reading_catalog_then_no_page_can_authorize_removal( + test_case: CatalogBoundaryTestCase, + tmp_path: Path, +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, "sqlite", "path") + adapter.close(connection) + # Exercise Databricks' real schema-only capability with a local read-only connection. + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-c", + "from pathlib import Path\n" + "import sys\n" + "from sqlbuild._native import run_location_command\n" + "from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter\n" + "from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter\n" + "SqliteAdapter.lists_relations_authoritatively = DatabricksAdapter.lists_relations_authoritatively\n" + "raise SystemExit(run_location_command(project_dir=Path(sys.argv[1]), " + "location_id='00000000-0000-0000-0000-000000000001', selected_target='development', " + "generation=1, server_url='http://127.0.0.1:1', action='runtime'))\n", + str(project), + ], + input=json.dumps( + { + "catalog": { + "location": { + "database": fixture["non_authoritative_database"], + "schema": "main", + } + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stderr + assert (json.loads(result.stdout) is not None) is test_case.expected_page_delivered + assert "No page delivered" in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogInspectionTestCase( + "DuckDB automatic catalog", "duckdb", "database", "BASE TABLE", "VIEW" + ), + CatalogInspectionTestCase( + "SQLite automatic catalog", "sqlite", "path", "BASE TABLE", "VIEW" + ), + ], + ids=lambda case: case.description, +) +def test_given_served_location_when_catalog_changes_then_facts_and_current_projection_follow_the_warehouse( + test_case: CatalogInspectionTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) + expected: dict[str, Any] = fixture["record"] | {"relation_type": test_case.expected_table_type} + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + wait_for_catalog(control, [expected], location, log) + assert [fact["present"] for fact in catalog_facts(control)] == [True] + time.sleep(6) + assert [fact["present"] for fact in catalog_facts(control)] == [True] + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute(connection=connection, sql=fixture["alter_sql"]) + adapter.close(connection) + expected = expected | { + "columns": [*expected["columns"], {"name": "shipped", "data_type": "INTEGER"}] + } + wait_for_catalog(control, [expected], location, log) + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + for statement in fixture["replace_sql"]: + _ = adapter.execute(connection=connection, sql=statement) + adapter.close(connection) + expected = expected | { + "relation_type": test_case.expected_view_type, + "columns": [{"name": "order_id", "data_type": "INTEGER"}], + } + wait_for_catalog(control, [expected], location, log) + assert catalog_facts(control)[-1]["record"]["relation_type"] == test_case.expected_view_type + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute(connection=connection, sql=fixture["drop_sql"]) + adapter.close(connection) + wait_for_catalog(control, [], location, log) + facts: list[dict[str, Any]] = catalog_facts(control) + assert [fact["present"] for fact in facts] == [True, True, True, False] + assert facts[-1]["record"] == expected + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + adapter.close(connection) + expected = fixture["record"] | {"relation_type": test_case.expected_table_type} + wait_for_catalog(control, [expected], location, log) + with closing(sqlite3.connect(control)) as database: + _ = database.execute("DELETE FROM sqb_warehouse_catalog") + database.commit() + wait_for_catalog(control, [expected], location, log) + assert catalog_facts(control)[-1]["record"] == expected From 659cef0e58ae0fc9c0b37c06af9e0d00fa9a4c07 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:43:41 +0000 Subject: [PATCH 099/138] feat: expose permission-enforced catalog pages --- .../orchestration/warehouse_catalog.json | 7 +- clients/typescript/src/schema.d.ts | 80 +++++ clients/typescript/tests/types.ts | 10 + .../authentication/password_sessions.rs | 4 + .../_helpers/resources/person_operations.rs | 29 ++ .../sqlbuild-orchestration/src/api/models.rs | 1 + .../warehouse_metadata/_helpers/catalog.rs | 79 ++++- .../src/warehouse_metadata/main/mod.rs | 1 + .../main/read_person_catalog.rs | 35 ++ .../src/warehouse_metadata/models.rs | 24 ++ .../tests/api/test_catalog_reads.rs | 324 ++++++++++++++++++ .../tests/api/test_contracts.rs | 2 + .../publication/person/helpers.py | 31 ++ .../publication/test_person_journey.py | 8 +- 14 files changed, 630 insertions(+), 5 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json index 524f87cdc6..b48ceea400 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json @@ -25,5 +25,10 @@ {"state_database": null, "state_schema": "main", "name": "Orders", "relation_type": "BASE TABLE", "columns": [{"name": "note", "data_type": "text"}, {"name": "shipped", "data_type": "boolean"}]}, {"state_database": null, "state_schema": "main", "name": "orders", "relation_type": "BASE TABLE", "columns": [{"name": "order_id", "data_type": "integer"}, {"name": "amount", "data_type": "numeric"}]} ], - "lifecycle": ["create", "unchanged scan", "column added", "table becomes view", "drop", "empty mirror rebuild"] + "lifecycle": ["create", "unchanged scan", "column added", "table becomes view", "drop", "empty mirror rebuild"], + "read_namespaces": [ + {"database": null, "schema": "main"}, + {"database": "orders_data", "schema": "main"}, + {"database": null, "schema": "reporting"} + ] } diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 43b8e7b9ad..7c83b11dd5 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -163,6 +163,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/catalog": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonCatalog"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -718,6 +734,17 @@ export interface components { /** Format: uuid */ session_id: string; }; + CatalogObservation: { + /** Format: uuid */ + event_id: string; + /** Format: date-time */ + observed_at: string; + record: components["schemas"]["WarehouseCatalogRecord"]; + }; + CatalogPage: { + next_after?: null | components["schemas"]["WarehouseReconciliationCursor"]; + records: components["schemas"]["CatalogObservation"][]; + }; CatchUp: "latest" | { /** @description Process the oldest bounded batch; later due times remain due for the next transaction. */ all: { @@ -2095,6 +2122,59 @@ export interface operations { }; }; }; + readPersonCatalog: { + parameters: { + query: { + /** @description Exact database; omitted matches SQL NULL */ + database?: string; + /** @description Exact physical schema */ + schema: string; + after_observed_at?: string; + after_event_id?: string; + /** @description Page size; defaults to 100, clamped to 1..100 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CatalogPage"]; + }; + }; + /** @description Invalid namespace or cursor */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 5b67c410b3..bbc8e9e8a4 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -86,6 +86,16 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/catalog // @ts-expect-error Catalog scans must include reconciliation; incremental imports cannot remove relations safely. const incompleteCatalog: components["schemas"]["CatalogBatch"] = { session_id: "synthetic-session", generation: 1, records: [] }; void incompleteCatalog; +void client.GET("/api/control/locations/{location_id}/targets/{target}/catalog", { + params: { + path: { location_id: "synthetic-location", target: "development" }, + query: { schema: "main", after_observed_at: "2026-01-02T03:04:05Z", after_event_id: "synthetic-relation", limit: 100 }, + }, +}); +const catalogPage: components["schemas"]["CatalogPage"] = { records: [], next_after: null }; +void catalogPage; +// @ts-expect-error Catalog reads require an exact physical schema. +void client.GET("/api/control/locations/{location_id}/targets/{target}/catalog", { params: { path: { location_id: "synthetic-location", target: "development" } } }); const sourceCondition: components["schemas"]["SensorCondition"] = { kind: "source_new_data", selection: ["source:raw_orders"], }; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 212171f999..0e93257eb7 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -62,6 +62,10 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/control/locations/{location_id}/targets/{target}/runs", get(crate::api::_helpers::resources::person_operations::recent_runs), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/catalog", + get(crate::api::_helpers::resources::person_operations::catalog), + ) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 43a89c8173..b90b926d89 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -17,6 +17,8 @@ use crate::runs::main::request_person_job_run::request_person_job_run; use crate::runs::main::request_rerun::request_rerun; use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord}; use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; +use crate::warehouse_metadata::models::{CatalogPage, CatalogQuery}; #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, @@ -106,3 +108,30 @@ pub(crate) async fn recent_runs( .await?, )) } + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/catalog", operation_id = "readPersonCatalog", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("database" = Option, Query, description = "Exact database; omitted matches SQL NULL"), + ("schema" = String, Query, description = "Exact physical schema"), + ("after_observed_at" = Option>, Query), + ("after_event_id" = Option, Query), + ("limit" = Option, Query, description = "Page size; defaults to 100, clamped to 1..100")), + responses((status = 200, body = CatalogPage), (status = 400, description = "Invalid namespace or cursor"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn catalog( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + headers: HeaderMap, +) -> Result, ApiError> { + Ok(Json( + read_person_catalog( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 28b74c5011..b9565b7cfd 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -20,6 +20,7 @@ use crate::api::_helpers::resources::{ person_operations::rerun, person_operations::change_trigger, person_operations::recent_runs, + person_operations::catalog, password_sessions::login, password_sessions::current, password_sessions::logout, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs index 6fe74655d9..49509fa406 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs @@ -7,9 +7,13 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; -use crate::store::models::{ControlEvent, ControlTransaction}; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; use crate::warehouse_metadata::_helpers::ingestion::validate_fields; -use crate::warehouse_metadata::models::CatalogBatch; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{ + CatalogBatch, CatalogObservation, CatalogPage, CatalogQuery, WarehouseReconciliationCursor, +}; pub(crate) fn validate_catalog(batch: &CatalogBatch) -> Result<(), ControlError> { let page = &batch.reconciliation; @@ -68,3 +72,74 @@ pub(crate) async fn append_catalog_fact( } Ok(transaction) } + +pub(crate) async fn read_catalog_page( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + options: &CatalogQuery, +) -> Result<(ControlTransaction, CatalogPage), ControlError> { + if options.schema.trim().is_empty() + || options.after_observed_at.is_some() != options.after_event_id.is_some() + { + return Err(ControlError::Invalid( + "catalog reads require a namespace and both cursor fields or neither".into(), + )); + } + let limit = options + .limit + .unwrap_or(100) + .clamp(1, MAX_HISTORY_RECORDS as u32) as usize; + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.schema.clone()), + ]; + let database = if let Some(database) = &options.database { + parameters.push(Parameter::Text(database.clone())); + "state_database=$4" + } else { + "state_database IS NULL" + }; + let mut after = String::new(); + if let (Some(time), Some(id)) = (options.after_observed_at, options.after_event_id) { + let offset = parameters.len(); + parameters.extend([Parameter::Time(time), Parameter::Id(id)]); + after = format!( + " AND (observed_at,event_id)>(${},${})", + offset + 1, + offset + 2 + ); + } + parameters.push(Parameter::Integer(limit as i64)); + let (transaction, rows) = query( + transaction, + &format!( + "SELECT event_id,observed_at,record_json AS record FROM sqb_warehouse_catalog \ + WHERE location_id=$1 AND target=$2 AND state_schema=$3 AND {database}{after} \ + ORDER BY observed_at,event_id LIMIT ${}", + parameters.len() + ), + parameters, + ) + .await?; + let more = rows.len() == limit; + let records: Vec = rows + .into_iter() + .map(|row| serde_json::from_value(json!(row))) + .collect::>() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let next_after = + more.then(|| records.last()) + .flatten() + .map(|record| WarehouseReconciliationCursor { + observed_at: record.observed_at, + event_id: record.event_id, + }); + Ok(( + transaction, + CatalogPage { + records, + next_after, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 6435a58598..b0fb90341e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -7,4 +7,5 @@ pub mod ingest_microbatches; pub mod ingest_migrations; pub mod ingest_old_name_views; pub mod ingest_source_freshness; +pub mod read_person_catalog; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs new file mode 100644 index 0000000000..edd9c30ec0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs @@ -0,0 +1,35 @@ +//! Live view permission and a bounded namespace page share the same transaction. + +use uuid::Uuid; + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_metadata::_helpers::catalog::read_catalog_page; +use crate::warehouse_metadata::models::{CatalogPage, CatalogQuery}; + +pub async fn read_person_catalog( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + (location_id, target, options): (Uuid, &str, &CatalogQuery), + default_role: DefaultRole, +) -> Result { + let (transaction, _) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + default_role, + ) + .await?; + let (transaction, page) = + read_catalog_page(transaction, (location_id, target), options).await?; + commit(transaction).await?; + Ok(page) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index ffed8ec562..1edd1123ce 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -524,3 +524,27 @@ pub struct CatalogRead { pub observed_since: Option>, pub limit: Option, } + +/// Read one exact physical namespace; an omitted database means SQL NULL, never a wildcard. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CatalogQuery { + pub database: Option, + pub schema: String, + pub after_observed_at: Option>, + pub after_event_id: Option, + pub limit: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CatalogObservation { + pub event_id: Uuid, + pub observed_at: DateTime, + pub record: WarehouseCatalogRecord, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CatalogPage { + pub records: Vec, + pub next_after: Option, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs new file mode 100644 index 0000000000..1400f381d3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -0,0 +1,324 @@ +//! Real ingestion, scoped person grants and namespace keyset pages on both control stores. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + create_principal::create_principal, login_password::login_password, set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorAuthentication, AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, + PasswordLogin, PrincipalKind, RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::{ + ingest_catalog::ingest_catalog, read_person_catalog::read_person_catalog, +}; +use crate::warehouse_metadata::models::{CatalogBatch, CatalogQuery}; + +#[test] +fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namespaces_bound_every_result() + { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite catalog reads", + backend: Backend::Sqlite, + expected_observations: 205, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL catalog reads", + backend: Backend::Postgres, + expected_observations: 205, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("local peer")), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = + create_principal(store, &context, ("orders-reader", PrincipalKind::Person)) + .await + .expect("person"); + let mut sessions = Vec::new(); + for (id, name) in [(admin, "orders-admin"), (person, "orders-reader")] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("authenticated"); + }; + sessions.push(session); + } + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json"); + let data: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("catalog inputs"); + let expected: Vec = (0..test_case.expected_observations) + .map(|index| format!("orders_🧾_{index:03}")) + .collect(); + for (scope, target) in [ + (location, "development"), + (location, "preview"), + (other, "development"), + ] { + let mut input = snapshot(scope); + input.target = target.into(); + publish_snapshot(store, &input).await.expect("snapshot"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + scope, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: target.into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("reader"); + for (namespace, count) in [ + (&data["read_namespaces"][0], test_case.expected_observations), + (&data["read_namespaces"][1], 1), + (&data["read_namespaces"][2], 1), + ] { + let records: Vec = (0..count) + .map(|index| { + let mut record = data["record"].clone(); + record["name"] = json!(format!("orders_🧾_{index:03}")); + record["state_database"] = namespace["database"].clone(); + record["state_schema"] = namespace["schema"].clone(); + record + }) + .collect(); + for records in records.chunks(100) { + let batch: CatalogBatch = serde_json::from_value(json!({"session_id":session_id,"generation":1,"records":records, + "reconciliation":{"scan_id":Uuid::now_v7(),"page":0,"state_database":namespace["database"],"state_schema":namespace["schema"],"day":null,"complete":false,"after":null}})).expect("batch"); + assert!( + ingest_catalog(store, (scope, target), &batch) + .await + .expect("real catalog ingestion") + .applied + ); + } + } + } + let credentials = (&context, (sessions[1].bearer.as_str(), "")); + let mut options = CatalogQuery { + database: None, + schema: "main".into(), + after_observed_at: None, + after_event_id: None, + limit: Some(u32::MAX), + }; + assert!(matches!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + let mut assignment = RoleAssignmentRequest { + principal_id: person, + role: BuiltinRole::Viewer, + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }; + assign_builtin_role( + store, + (&context, (&sessions[0].bearer, &sessions[0].csrf)), + &assignment, + &AdministratorAuthentication::Password, + ) + .await + .expect("scoped viewer"); + let mut actual = Vec::new(); + for _ in 0..2 { + let page = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("bounded page"); + assert_eq!(page.records.len(), 100, "{}", test_case.description); + let after = page.next_after.expect("more rows remain"); + options.after_observed_at = Some(after.observed_at); + options.after_event_id = Some(after.event_id); + actual.extend(page.records); + } + let last = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("last page"); + assert_eq!(last.records.len(), 5); + assert!(last.next_after.is_none()); + actual.extend(last.records); + assert_eq!(actual.len(), test_case.expected_observations); + for row in &actual { + assert_eq!(json!(row.record.columns), data["record"]["columns"]); + } + let mut names = actual + .into_iter() + .map(|row| row.record.name) + .collect::>(); + names.sort(); + assert_eq!( + names, expected, + "ties and namespace exclusions preserve each exact name once" + ); + options.after_observed_at = None; + options.after_event_id = None; + options.limit = Some(0); + assert_eq!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("minimum page") + .records + .len(), + 1 + ); + options.database = Some("orders_data".into()); + let exact = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("exact database"); + assert_eq!(exact.records.len(), 1); + assert_eq!( + exact.records[0].record.state_database.as_deref(), + Some("orders_data") + ); + let after = exact.next_after.expect("full page needs continuation"); + options.after_observed_at = Some(after.observed_at); + options.after_event_id = Some(after.event_id); + let empty = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("exact multiple ends with empty page"); + assert!(empty.records.is_empty()); + assert!(empty.next_after.is_none()); + options.after_observed_at = None; + options.after_event_id = None; + for scope in [(other, "development"), (location, "preview")] { + assert!(matches!( + read_person_catalog( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + } + options.after_event_id = Some(Uuid::now_v7()); + assert!(matches!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Invalid(_)) + )); + assignment.revoke = true; + assign_builtin_role( + store, + (&context, (&sessions[0].bearer, &sessions[0].csrf)), + &assignment, + &AdministratorAuthentication::Password, + ) + .await + .expect("revoke"); + assert!( + matches!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + ), + "authorization precedes even invalid cursor handling" + ); + options.after_event_id = None; + assert_eq!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::Viewer + ) + .await + .expect("configured viewer default") + .records + .len(), + 1 + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 2bead6b842..27c920acaa 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -12,6 +12,8 @@ mod oidc_test_types; mod output_helpers; #[path = "recall/helpers.rs"] mod recall_helpers; +#[path = "test_catalog_reads.rs"] +mod test_catalog_reads; #[path = "test_changes.rs"] mod test_changes; #[path = "test_contract.rs"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index e2ce99fe97..0ff5a3c3de 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -4,8 +4,10 @@ import os import subprocess import sys +import time from http.cookies import SimpleCookie from pathlib import Path +from subprocess import Popen from typing import Any from urllib.error import HTTPError from urllib.request import Request, urlopen @@ -59,6 +61,35 @@ def person_read(api: SnapshotApi, path: str, *, session: tuple[str, str]) -> Any return json.load(response) +def wait_for_person_catalog( + api: SnapshotApi, session: tuple[str, str], process: Popen[bytes], log: Path +) -> None: + """Wait for an actual worker-built relation through the scoped browser API, not store SQL.""" + path: str = f"/api/control/locations/{api.location_id}/targets/development/catalog?schema=main" + deadline: float = time.monotonic() + 45 + page: dict[str, Any] = person_read(api, path, session=session) + records: dict[str, Any] = {row["record"]["name"]: row for row in page["records"]} + while "orders" not in records: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (page, log.read_text()) + time.sleep(0.05) + page = person_read(api, path, session=session) + records = {row["record"]["name"]: row for row in page["records"]} + assert records["orders"]["event_id"] and records["orders"]["observed_at"] + assert records["orders"]["record"] == { + "state_database": None, + "state_schema": "main", + "name": "orders", + "relation_type": "BASE TABLE", + "columns": [{"name": "amount", "data_type": "INTEGER"}], + } + assert page["next_after"] is None + assert person_read(api, path + "&database=unlisted", session=session) == { + "records": [], + "next_after": None, + } + + def verify_security_audit( api: SnapshotApi, administrator: tuple[str, str], ordinary: tuple[str, str] ) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index ab522a37e7..683fda72d1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -31,6 +31,7 @@ person_request, provision_person, verify_security_audit, + wait_for_person_catalog, ) @@ -230,12 +231,15 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi recovery: dict[str, Any] = wait_for_run( worker.api, worker.process, worker.log, run_id=request["run_id"] ) - assert recovery["status"] == test_case.expected_status - assert recovery["snapshot_id"] == recovered["record"]["snapshot_id"] + assert (recovery["status"], recovery["snapshot_id"]) == ( + test_case.expected_status, + recovered["record"]["snapshot_id"], + ) with duckdb.connect( str(worker.project / "warehouse.duckdb"), read_only=True ) as warehouse: assert warehouse.sql(test_case.warehouse_query).fetchall() == [(41,)] + wait_for_person_catalog(worker.api, session, location, log) assignment["revoke"] = True assert person_request( worker.api, "/api/admin/roles", body=assignment, session=admin_session From 9ab16a1a8b0530f9b0cabd31dbef4aa96bb93e9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:46:56 +0000 Subject: [PATCH 100/138] feat: expose bounded latest build fingerprint evidence --- .../orchestration/fingerprint_lookup.json | 21 +++ clients/typescript/src/schema.d.ts | 81 +++++++++ clients/typescript/tests/types.ts | 6 + .../authentication/password_sessions.rs | 4 + .../_helpers/resources/person_operations.rs | 31 +++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/store/_helpers/migration.rs | 1 + .../src/store/fingerprint_lookup.sql | 13 ++ .../src/store/main/migrate_store.rs | 14 +- .../src/warehouse_metadata/_helpers/mod.rs | 1 + .../_helpers/person_reads.rs | 23 +++ .../main/ingest_fingerprints.rs | 8 +- .../src/warehouse_metadata/main/mod.rs | 1 + .../main/read_person_catalog.rs | 10 +- .../main/read_person_fingerprint.rs | 63 +++++++ .../src/warehouse_metadata/models.rs | 23 +++ .../tests/api/fingerprint_reads/helpers.rs | 167 ++++++++++++++++++ .../tests/api/test_catalog_reads.rs | 23 +++ .../tests/api/test_contracts.rs | 2 + .../tests/credentials/test_audit.rs | 1 + .../tests/sensors/test_sources.rs | 1 + .../tests/store/helpers.rs | 23 +++ .../publication/person/helpers.py | 21 +++ 23 files changed, 524 insertions(+), 15 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json create mode 100644 crates/sqlbuild-orchestration/src/store/fingerprint_lookup.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json new file mode 100644 index 0000000000..4849efbf6c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json @@ -0,0 +1,21 @@ +{ + "record": { + "event_id": "orders-build-latest", + "state_database": null, + "state_schema": "build_history", + "node_type": "model", + "node_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders", + "run_id": "orders-run-latest", + "definition_hash": "orders-definition-latest", + "version_hash": "orders-version-latest", + "schema_fingerprint": "orders-schema-latest", + "definition": "SELECT order_id FROM orders", + "metadata_json": "{\"private_note\":\"not part of the person response\"}", + "ts": "2026-01-03T10:00:00Z" + }, + "older_ts": "2026-01-01T10:00:00Z", + "distractor_ts": "2026-01-04T10:00:00Z" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 7c83b11dd5..fea1bb51af 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -179,6 +179,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/fingerprint": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonFingerprint"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -809,6 +825,18 @@ export interface components { /** Format: uuid */ session_id: string; }; + /** @description Build evidence only: no SQL, arbitrary metadata, or inference about live relation existence. */ + FingerprintObservation: { + definition_hash: string; + /** Format: uuid */ + event_id: string; + /** Format: date-time */ + observed_at: string; + run_id: string; + schema_fingerprint: string; + source_event_id: string; + version_hash: string; + }; HeartbeatResponse: { cancel_requested: boolean; }; @@ -2175,6 +2203,59 @@ export interface operations { }; }; }; + readPersonFingerprint: { + parameters: { + query: { + node_type: string; + node_name: string; + /** @description Exact destination database; omitted matches SQL NULL */ + database?: string; + /** @description Exact destination schema; omitted matches SQL NULL */ + schema?: string; + /** @description Exact destination name; omitted matches SQL NULL */ + name?: string; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["FingerprintObservation"]; + }; + }; + /** @description Invalid node */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index bbc8e9e8a4..07e2d4ee21 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -116,3 +116,9 @@ void client.POST("/api/auth/logout", {}); void client.POST("/api/auth/password", { params: { header: { "x-sqb-csrf": "synthetic-csrf" } }, body: { new_password: "synthetic-password" } }); // @ts-expect-error Session responses never expose a browser-readable bearer. void ({} as components["schemas"]["PasswordSessionResponse"]).bearer; +void client.GET("/api/control/locations/{location_id}/targets/{target}/fingerprint", { + params: { path: { location_id: "orders-location", target: "development" }, + query: { node_type: "model", node_name: "orders", schema: "main", name: "orders" } }, +}); +// @ts-expect-error Build evidence does not contain SQL or arbitrary metadata. +void ({} as components["schemas"]["FingerprintObservation"]).definition; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 0e93257eb7..54051fd839 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -66,6 +66,10 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/control/locations/{location_id}/targets/{target}/catalog", get(crate::api::_helpers::resources::person_operations::catalog), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/fingerprint", + get(crate::api::_helpers::resources::person_operations::fingerprint), + ) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index b90b926d89..0de02072dc 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -18,7 +18,10 @@ use crate::runs::main::request_rerun::request_rerun; use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord}; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; -use crate::warehouse_metadata::models::{CatalogPage, CatalogQuery}; +use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; +use crate::warehouse_metadata::models::{ + CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, +}; #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, @@ -135,3 +138,29 @@ pub(crate) async fn catalog( .await?, )) } + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/fingerprint", operation_id = "readPersonFingerprint", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("node_type" = String, Query), ("node_name" = String, Query), + ("database" = Option, Query, description = "Exact destination database; omitted matches SQL NULL"), + ("schema" = Option, Query, description = "Exact destination schema; omitted matches SQL NULL"), + ("name" = Option, Query, description = "Exact destination name; omitted matches SQL NULL")), + responses((status = 200, body = Option), (status = 400, description = "Invalid node"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn fingerprint( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + headers: HeaderMap, +) -> Result>, ApiError> { + Ok(Json( + read_person_fingerprint( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index b9565b7cfd..0caf4e5d7d 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -21,6 +21,7 @@ use crate::api::_helpers::resources::{ person_operations::change_trigger, person_operations::recent_runs, person_operations::catalog, + person_operations::fingerprint, password_sessions::login, password_sessions::current, password_sessions::logout, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 4c8fa57cc8..c9a42eb73a 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -35,6 +35,7 @@ pub(crate) const SOURCE_SENSORS: &str = include_str!("../source_sensors.sql"); pub(crate) const SOURCE_STALENESS: &str = include_str!("../source_staleness.sql"); pub(crate) const SECURITY_AUDIT_SINK: &str = include_str!("../security_audit_sink.sql"); pub(crate) const WAREHOUSE_CATALOG: &str = include_str!("../warehouse_catalog.sql"); +pub(crate) const FINGERPRINT_LOOKUP: &str = include_str!("../fingerprint_lookup.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/fingerprint_lookup.sql b/crates/sqlbuild-orchestration/src/store/fingerprint_lookup.sql new file mode 100644 index 0000000000..b26dc2f775 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/fingerprint_lookup.sql @@ -0,0 +1,13 @@ +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN node_type TEXT; +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN node_name TEXT; +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN target_database TEXT; +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN target_schema TEXT; +ALTER TABLE sqb_warehouse_fingerprints ADD COLUMN target_name TEXT; +UPDATE sqb_warehouse_fingerprints SET + node_type=record_json ->> 'node_type', + node_name=record_json ->> 'node_name', + target_database=record_json ->> 'target_database', + target_schema=record_json ->> 'target_schema', + target_name=record_json ->> 'target_name'; +CREATE INDEX sqb_fingerprint_node_latest ON sqb_warehouse_fingerprints + (location_id,target,node_type,node_name,target_database,target_schema,target_name,observed_at DESC,event_id DESC); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index e76623dec5..51c4bb1067 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -5,12 +5,13 @@ use uuid::Uuid; use crate::store::_helpers::migration::{ ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, - DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, LOCATION_INSTANCES, LOCATION_WORK, - NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, - PLATFORM_SINK, RUN_SENSORS, SCHEMA, SECURITY_AUDIT_SINK, SENSORS, SOURCE_SENSORS, - SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_CATALOG, WAREHOUSE_FINGERPRINTS, - WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, - WAREHOUSE_RECONCILIATION, render, verify_migrations, + DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, FINGERPRINT_LOOKUP, + LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, + PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, + SECURITY_AUDIT_SINK, SENSORS, SOURCE_SENSORS, SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, + WAREHOUSE_CATALOG, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, + WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, + verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -62,6 +63,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("source_staleness", SOURCE_STALENESS), ("security_audit_sink", SECURITY_AUDIT_SINK), ("warehouse_catalog", WAREHOUSE_CATALOG), + ("fingerprint_lookup", FINGERPRINT_LOOKUP), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs index 08f156dd76..e3204dbb07 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod events; pub(crate) mod fingerprints; pub(crate) mod freshness; pub(crate) mod ingestion; +pub(crate) mod person_reads; pub(crate) mod reconciliation; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs new file mode 100644 index 0000000000..75f57a1558 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs @@ -0,0 +1,23 @@ +//! Person metadata reads share a live permission decision and the operation's writer fence. + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn begin_view( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + scope: &PermissionScope, + default_role: DefaultRole, +) -> Result { + let (transaction, _) = authorize_session( + begin(store, TransactionMode::Write).await?, + credentials, + scope, + default_role, + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs index 150d27f615..9346fa71aa 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs @@ -79,11 +79,13 @@ pub async fn ingest_fingerprints( .await?; let inserted; (transaction, inserted) = query(transaction, - "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema,node_type,node_name,target_database,target_schema,target_name) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), - Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; + Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Text(record.node_type.clone()), Parameter::Text(record.node_name.clone()), Parameter::OptionalText(record.target_database.clone()), + Parameter::OptionalText(record.target_schema.clone()), Parameter::OptionalText(record.target_name.clone())]).await?; applied |= !inserted.is_empty(); } } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index b0fb90341e..f3355f9c82 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -8,4 +8,5 @@ pub mod ingest_migrations; pub mod ingest_old_name_views; pub mod ingest_source_freshness; pub mod read_person_catalog; +pub mod read_person_fingerprint; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs index edd9c30ec0..25a7e38a4d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs @@ -2,12 +2,12 @@ use uuid::Uuid; -use crate::authorization::main::authorize_session::authorize_session; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit}; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::catalog::read_catalog_page; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; use crate::warehouse_metadata::models::{CatalogPage, CatalogQuery}; pub async fn read_person_catalog( @@ -16,8 +16,8 @@ pub async fn read_person_catalog( (location_id, target, options): (Uuid, &str, &CatalogQuery), default_role: DefaultRole, ) -> Result { - let (transaction, _) = authorize_session( - begin(store, TransactionMode::Write).await?, + let transaction = begin_view( + store, (context, credentials), &PermissionScope { action: Action::View, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs new file mode 100644 index 0000000000..2d2ca1660d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs @@ -0,0 +1,63 @@ +//! Read one exact latest build through the node/destination index, never infer live state. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; + +pub async fn read_person_fingerprint( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location_id, target, options): (Uuid, &str, &FingerprintQuery), + default_role: DefaultRole, +) -> Result, ControlError> { + let transaction = begin_view( + store, + credentials, + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [format!("{}:{}", options.node_type, options.node_name)].into(), + }, + default_role, + ) + .await?; + if options.node_type.trim().is_empty() || options.node_name.trim().is_empty() { + return Err(ControlError::Invalid( + "fingerprint reads require an exact node".into(), + )); + } + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.node_type.clone()), + Parameter::Text(options.node_name.clone()), + ]; + let mut predicates: Vec = Vec::new(); + for (column, value) in [ + ("target_database", &options.database), + ("target_schema", &options.schema), + ("target_name", &options.name), + ] { + predicates.push(if let Some(value) = value { + parameters.push(Parameter::Text(value.clone())); + format!("{column}=${}", parameters.len()) + } else { + format!("{column} IS NULL") + }); + } + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,observed_at,source_event_id,record_json ->> 'run_id' AS run_id, \ + record_json ->> 'definition_hash' AS definition_hash,record_json ->> 'version_hash' AS version_hash, \ + record_json ->> 'schema_fingerprint' AS schema_fingerprint FROM sqb_warehouse_fingerprints \ + WHERE location_id=$1 AND target=$2 AND node_type=$3 AND node_name=$4 AND {} \ + ORDER BY target_database,target_schema,target_name,observed_at DESC,event_id DESC LIMIT 1", predicates.join(" AND ")), parameters).await?; + let record = rows.into_iter().next().map(decode_row).transpose()?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 1edd1123ce..8cfc89d70c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -548,3 +548,26 @@ pub struct CatalogPage { pub records: Vec, pub next_after: Option, } + +/// Exact node and physical destination, not a selector; omitted namespace values match SQL NULL. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FingerprintQuery { + pub node_type: String, + pub node_name: String, + pub database: Option, + pub schema: Option, + pub name: Option, +} + +/// Build evidence only: no SQL, arbitrary metadata, or inference about live relation existence. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct FingerprintObservation { + pub event_id: Uuid, + pub observed_at: DateTime, + pub source_event_id: String, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, +} diff --git a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs new file mode 100644 index 0000000000..448710fae3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs @@ -0,0 +1,167 @@ +//! Actual ingestion and additive upgrade retain bounded, exact-destination build evidence. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; +use crate::store::tests::store_helpers::reconstruct_fingerprint_predecessor; +use crate::warehouse_metadata::main::{ + ingest_fingerprints::ingest_fingerprints, read_person_fingerprint::read_person_fingerprint, +}; +use crate::warehouse_metadata::models::{FingerprintBatch, FingerprintQuery}; + +pub(crate) async fn fingerprint_reads( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location, other): (Uuid, Uuid), +) { + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("reader session"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json"); + let data: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("fingerprint inputs"); + let latest = data["record"].clone(); + let mut older = latest.clone(); + older["event_id"] = json!("orders-build-older"); + older["ts"] = data["older_ts"].clone(); + older["version_hash"] = json!("orders-version-older"); + let mut records = vec![latest.clone(), older]; + for (key, value) in [ + ("target_database", json!("orders_data")), + ("target_schema", json!("landing")), + ("target_name", json!("Orders")), + ("node_type", json!("source")), + ("node_name", json!("customers")), + ] { + let mut record = latest.clone(); + record["event_id"] = json!(format!("orders-distractor-{key}")); + record["ts"] = data["distractor_ts"].clone(); + record[key] = value; + records.push(record); + } + let batch: FingerprintBatch = serde_json::from_value(json!({ + "session_id":session_id,"generation":1,"records":records, + })) + .expect("batch"); + assert!( + ingest_fingerprints(store, (location, "development"), &batch) + .await + .expect("ingest") + .applied + ); + let mut options = FingerprintQuery { + node_type: "model".into(), + node_name: "orders".into(), + database: None, + schema: Some("main".into()), + name: Some("orders".into()), + }; + let first = read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("latest build") + .expect("evidence"); + let serialized = json!(&first); + assert_eq!( + serialized, + json!({ + "event_id":first.event_id,"observed_at":latest["ts"],"source_event_id":latest["event_id"], + "run_id":latest["run_id"],"definition_hash":latest["definition_hash"], + "version_hash":latest["version_hash"],"schema_fingerprint":latest["schema_fingerprint"], + }) + ); + for scope in [(other, "development"), (location, "preview")] { + assert!(matches!( + read_person_fingerprint( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + read_person_fingerprint( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::Viewer, + ) + .await + .expect("authorized empty scope") + .is_none() + ); + } + options.schema = None; + assert!( + read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("NULL schema is exact") + .is_none() + ); + options.schema = Some("main".into()); + options.name = None; + assert!( + read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("NULL relation is exact") + .is_none() + ); + options.name = Some("orders".into()); + reconstruct_fingerprint_predecessor(store).await; + migrate_store(store) + .await + .expect("populated fingerprint upgrade"); + migrate_store(store).await.expect("idempotent migration"); + assert!( + !ingest_fingerprints(store, (location, "development"), &batch) + .await + .expect("replay") + .applied + ); + assert_eq!( + json!( + read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("backfilled index") + ), + serialized + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index 1400f381d3..ddab9dc6de 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -174,6 +174,12 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp ) .await .expect("scoped viewer"); + crate::api::tests::fingerprint_reads_helpers::fingerprint_reads( + store, + credentials, + (location, other), + ) + .await; let mut actual = Vec::new(); for _ in 0..2 { let page = read_person_catalog( @@ -291,6 +297,23 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp ) .await .expect("revoke"); + let evidence = crate::warehouse_metadata::models::FingerprintQuery { + node_type: "model".into(), + node_name: "orders".into(), + database: None, + schema: Some("main".into()), + name: Some("orders".into()), + }; + assert!(matches!( + crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint( + store, + credentials, + (location, "development", &evidence), + DefaultRole::None, + ) + .await, + Err(ControlError::Denied { .. }) + )); assert!( matches!( read_person_catalog( diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 27c920acaa..7736c079fd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "changes/helpers.rs"] mod changes_helpers; +#[path = "fingerprint_reads/helpers.rs"] +mod fingerprint_reads_helpers; #[path = "helpers.rs"] pub(crate) mod helpers; #[path = "oidc/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 6c8a9161d5..175c166145 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -133,6 +133,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .map(|event| (event.occurred_at, event.event_id)) .collect::>(); expected.sort(); + crate::store::tests::store_helpers::reconstruct_fingerprint_predecessor(store).await; let mut transaction = begin(store, TransactionMode::Write) .await .expect("predecessor"); diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 0817e93771..c8f655b645 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -169,6 +169,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .applied ); } + crate::store::tests::store_helpers::reconstruct_fingerprint_predecessor(store).await; let mut transaction = begin(store, TransactionMode::Write) .await .expect("pre-sensor predecessor"); diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index a3a067fe17..7feba25190 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -75,7 +75,30 @@ pub(crate) async fn close_store(store: ControlStore) { pool.close().await; } +pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("fingerprint predecessor"); + for sql in [ + "DROP INDEX sqb_fingerprint_node_latest", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN node_type", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN node_name", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN target_database", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN target_schema", + "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN target_name", + "DELETE FROM sqb_schema_migrations WHERE name='fingerprint_lookup'", + ] { + (transaction, _) = query(transaction, sql, vec![]) + .await + .expect("fingerprint predecessor"); + } + commit(transaction) + .await + .expect("commit fingerprint predecessor"); +} + pub(crate) async fn reconstruct_warehouse_predecessor(store: &ControlStore) { + reconstruct_fingerprint_predecessor(store).await; let mut transaction = begin(store, TransactionMode::Write) .await .expect("predecessor"); diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 0ff5a3c3de..8d1f8466f0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -88,6 +88,27 @@ def wait_for_person_catalog( "records": [], "next_after": None, } + fingerprint_path: str = ( + f"/api/control/locations/{api.location_id}/targets/development/fingerprint" + "?node_type=model&node_name=orders&schema=main&name=orders" + ) + evidence: dict[str, Any] | None = person_read(api, fingerprint_path, session=session) + while evidence is None: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.05) + evidence = person_read(api, fingerprint_path, session=session) + assert set(evidence) == { + "event_id", + "observed_at", + "source_event_id", + "run_id", + "definition_hash", + "version_hash", + "schema_fingerprint", + } + assert all(evidence.values()), evidence + assert person_read(api, fingerprint_path + "&database=unlisted", session=session) is None def verify_security_audit( From 2e6ef6a091511aedd458fae451221e421c15df26 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:03:36 +0000 Subject: [PATCH 101/138] ci: separate native workspace verification --- .github/workflows/verify.yml | 48 +++++++++++++++--- .../release_gate/_helpers/_test_types.py | 9 ++++ .../release_gate/_helpers/test_gate.py | 49 +++++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 8f4426a27d..08ae232070 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -293,8 +293,6 @@ jobs: uv run --no-sync npm --prefix clients/typescript test - name: Run static checks run: make check-ci - - name: Run native checks - run: make rust-check - name: Prepare Docker worker tests run: | docker info > /dev/null @@ -304,6 +302,38 @@ jobs: SQLBUILD_REQUIRE_DOCKER_TESTS: "1" run: make test + native_checks: + name: Native workspace checks + needs: [changes, lockfile] + if: needs.changes.outputs.checks == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: verify-checks + save-if: false + - name: Install Python + run: uv python install 3.12 + - name: Install native test dependencies without rebuilding the packages + run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - name: Run native checks + env: + UV_NO_SYNC: "1" + PYO3_PYTHON: ${{ github.workspace }}/.venv/bin/python + run: make rust-check + e2e_shards: name: DuckDB E2E (${{ matrix.name }}) needs: [changes, lockfile] @@ -1030,6 +1060,7 @@ jobs: changes, lockfile, checks, + native_checks, e2e_shards, compiler_differential, compiler_differential_python_versions, @@ -1054,6 +1085,7 @@ jobs: CHECKS_SELECTED: ${{ needs.changes.outputs.checks }} LOCKFILE_RESULT: ${{ needs.lockfile.result }} CHECKS_RESULT: ${{ needs.checks.result }} + NATIVE_CHECKS_RESULT: ${{ needs.native_checks.result }} E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }} @@ -1074,11 +1106,13 @@ jobs: echo "Required classification or lockfile checks did not pass" >&2 exit 1 fi - if [[ "${CHECKS_RESULT}" != "success" ]] && \ - ! [[ "${CHECKS_SELECTED}" == "false" && "${CHECKS_RESULT}" == "skipped" ]]; then - echo "Required static checks did not pass: ${CHECKS_RESULT}" >&2 - exit 1 - fi + for result in "${CHECKS_RESULT}" "${NATIVE_CHECKS_RESULT}"; do + if [[ "${result}" != "success" ]] && \ + ! [[ "${CHECKS_SELECTED}" == "false" && "${result}" == "skipped" ]]; then + echo "Required static or native checks did not pass: ${result}" >&2 + exit 1 + fi + done if [[ "${WHEELS_RESULT}" == "success" && "${COMPOSE_RESULT}" != "success" ]]; then echo "Required packaged Compose verification did not pass: ${COMPOSE_RESULT}" >&2 exit 1 diff --git a/tests/unit/scripts/release_gate/_helpers/_test_types.py b/tests/unit/scripts/release_gate/_helpers/_test_types.py index 0792f1eb63..a1e9dc6c9e 100644 --- a/tests/unit/scripts/release_gate/_helpers/_test_types.py +++ b/tests/unit/scripts/release_gate/_helpers/_test_types.py @@ -48,6 +48,15 @@ class RequiredCheckTestCase: expected_passes: bool +@dataclass(frozen=True) +class VerificationSuitesTestCase: + description: str + selected: str + checks: str + native_checks: str + expected_exit_code: int + + @dataclass(frozen=True) class ExpressionTestCase: description: str diff --git a/tests/unit/scripts/release_gate/_helpers/test_gate.py b/tests/unit/scripts/release_gate/_helpers/test_gate.py index d43aa66213..1a291b3b67 100644 --- a/tests/unit/scripts/release_gate/_helpers/test_gate.py +++ b/tests/unit/scripts/release_gate/_helpers/test_gate.py @@ -1,4 +1,7 @@ +import os +import subprocess from pathlib import Path +from typing import Any import pytest @@ -7,6 +10,7 @@ from tests.unit.scripts.release_gate._helpers._test_types import ( GateViolationTestCase, RepositoryGateTestCase, + VerificationSuitesTestCase, VerifyCheckNameTestCase, ) from tests.unit.scripts.release_gate._helpers.helpers import ( @@ -164,5 +168,50 @@ def test_given_workflows_when_checking_release_gate_then_reports_violations( assert get_release_gate_errors(tmp_path) == test_case.expected_errors +@pytest.mark.parametrize( + "test_case", + [ + VerificationSuitesTestCase("both selected suites pass", "true", "success", "success", 0), + VerificationSuitesTestCase( + "native failure cannot be masked by static success", "true", "success", "failure", 1 + ), + VerificationSuitesTestCase( + "selected native skip fails closed", "true", "success", "skipped", 1 + ), + VerificationSuitesTestCase( + "native cancellation fails closed", "true", "success", "cancelled", 1 + ), + VerificationSuitesTestCase( + "static failure cannot be masked by native success", "true", "failure", "success", 1 + ), + VerificationSuitesTestCase( + "docs-only skips terminate successfully", "false", "skipped", "skipped", 0 + ), + ], + ids=lambda case: case.description, +) +def test_given_selected_suites_when_aggregating_then_failures_and_skips_resolve( + test_case: VerificationSuitesTestCase, +) -> None: + job: dict[str, Any] = read_verify_job() + assert "native_checks" in job["needs"] + step: dict[str, Any] = job["steps"][0] + environment: dict[str, str] = {name: "success" for name in step["env"]} + environment.update( + CHECKS_SELECTED=test_case.selected, + CHECKS_RESULT=test_case.checks, + NATIVE_CHECKS_RESULT=test_case.native_checks, + ) + result: subprocess.CompletedProcess[str] = subprocess.run( + ["bash", "-e", "-c", step["run"]], + env=os.environ | environment, + capture_output=True, + text=True, + check=False, + timeout=5, + ) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From c1ae966d6b0665d9cbf4900f910c7c2c29484fa5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:22:25 +0000 Subject: [PATCH 102/138] feat: publish planner-owned snapshot identities --- .../orchestration/semantic_identity.json | 57 +++++++++ .../_helpers/compiled_project/snapshot.rs | 76 +----------- .../_helpers/location_command/command.rs | 6 +- .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/snapshot.rs | 113 ++++++++++++++++++ .../compiler/compiled_project/_test_types.py | 9 ++ .../test_orchestration_snapshot.py | 64 ++++++++++ 7 files changed, 250 insertions(+), 76 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json b/benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json new file mode 100644 index 0000000000..5b1c5e0140 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json @@ -0,0 +1,57 @@ +{ + "files": { + "hooks/python/notifications.py": "from sqlbuild.hooks import hook\n\n@hook\ndef notify(ctx, channel):\n \"\"\"Notify support.\"\"\"\n return 'orders'\n", + "models/orders.sql": "MODEL (description 'Orders.', materialized 'table', pre_hooks [python('notify', channel: 'support')], columns (order_id (type INTEGER), amount (type BIGINT)));\nSELECT order_id, __udf(\"edge_next_quantity\")(amount) AS amount FROM __ref(\"stg_orders\")\n" + }, + "cases": [ + { + "description": "layout does not change semantic identity", + "path": "models/orders.sql", + "before": "SELECT order_id,", + "after": "-- Order presentation\nSELECT\n order_id,", + "expected_changed": [] + }, + { + "description": "config changes with identical query", + "path": "models/orders.sql", + "before": "materialized 'table'", + "after": "materialized 'view'", + "expected_changed": ["model:orders"] + }, + { + "description": "hook code changes with identical query and config", + "path": "hooks/python/notifications.py", + "before": "return 'orders'", + "after": "return 'products'", + "expected_changed": ["model:orders"] + }, + { + "description": "referenced function changes consumer local identity", + "path": "functions/sql/edge_cases/edge_next_quantity.sql", + "before": "quantity + 1", + "after": "quantity + 3", + "expected_changed": ["udf:edge_next_quantity", "model:orders"] + }, + { + "description": "upstream query changes only its local identity", + "path": "models/stg_orders.sql", + "before": "CAST(5 AS INTEGER)", + "after": "CAST(9 AS INTEGER)", + "expected_changed": ["model:stg_orders"] + }, + { + "description": "seed data changes its planner content identity", + "path": "seeds/edge_cases/edge_quantity_steps.csv", + "before": "\n7\n", + "after": "\n11\n", + "expected_changed": ["seed:edge_quantity_steps"] + }, + { + "description": "CSV layout is not seed semantic data", + "path": "seeds/edge_cases/edge_quantity_steps.csv", + "before": "\n7\n", + "after": "\n\"7\"\n", + "expected_changed": [] + } + ] +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs index 8edfb3a580..ff338a5e1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -1,29 +1,16 @@ //! Project metadata stays native until its immutable snapshot is sent over HTTP. use std::collections::BTreeMap; -use std::path::PathBuf; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; use serde_json::Value; -use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - compile_declarations, discover_declarations, -}; -use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; -use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; -use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; -use crate::bindings::_helpers::records::fields::field; /// A data-only publication; this object never contains adapters, connections or user callables. #[pyclass(frozen, module = "sqlbuild._native")] @@ -82,62 +69,6 @@ impl NativeOrchestrationSnapshot { } } -/// Compile catalog nodes from their owners, never from Python model or graph projections. -#[pyfunction] -#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] -pub(crate) fn build_orchestration_snapshot( - project: PyRef<'_, NativeProjectView>, - metadata_json: &str, - project_dir: PathBuf, - declarations: Option>>, -) -> PyResult { - let py = project.py(); - let mut input: SnapshotInput = serde_json::from_str(metadata_json) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - if !input.nodes.is_empty() { - return Err(PyValueError::new_err( - "snapshot nodes must come from the compiled project", - )); - } - if project - .effective_target_name() - .is_some_and(|target| target != input.target) - { - return Err(PyValueError::new_err( - "snapshot target differs from the compiled project", - )); - } - for diagnostic in project.diagnostics(py).bind(py).try_iter()? { - let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; - if severity == DiagnosticSeverity::Error.as_str() { - return Err(PyValueError::new_err( - "cannot publish a project with compiler errors", - )); - } - } - input.sqb_version = env!("CARGO_PKG_VERSION").into(); - input.environment_hash = environment_hash(py, &project_dir)?; - let owner = project.owner().bind(py).borrow(); - input.nodes = model_nodes(py, &project, &owner, input.nodes)?; - input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; - drop(owner); - let declarations = match declarations { - Some(declarations) => declarations, - None => discover_declarations(py, &project_dir)?, - }; - if !declarations.is_empty() { - let authored = collect_declarations(py, declarations)?; - let graph = project.lineage_graph(py)?; - input = compile_declarations( - py, - input, - (&project_dir, graph.bind(py).get().graph()), - authored, - )?; - } - Ok(NativeOrchestrationSnapshot { input }) -} - /// Duplicate resource identities must fail, rather than silently replacing an earlier node. pub(crate) fn insert_node( mut nodes: BTreeMap, @@ -169,6 +100,5 @@ pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?) + module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index e46264c2ac..dfd9e0eb1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -18,12 +18,11 @@ use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; -use crate::bindings::_helpers::compiled_project::snapshot::{ - NativeOrchestrationSnapshot, build_orchestration_snapshot, -}; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; +use crate::bindings::_helpers::location_command::snapshot::build_orchestration_snapshot; use crate::bindings::_helpers::location_command::worker::execute_worker; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ @@ -317,5 +316,6 @@ pub(crate) fn run_location<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?)?; module.add_function(wrap_pyfunction!(run_location_command, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 63f708bfc2..79131a3bb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod command; mod constants; mod runtime; +mod snapshot; mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs new file mode 100644 index 0000000000..f975aa24ed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -0,0 +1,113 @@ +//! Location publication assembles catalog data and planner-owned local identities. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{IntoPyObject, pyfunction}; +use serde_json::json; +use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; + +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ + compile_declarations, discover_declarations, +}; +use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; +use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; +use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; +use crate::bindings::_helpers::records::fields::field; + +/// Compile catalog nodes and identities in the location, never in the server. +#[pyfunction] +#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] +pub(crate) fn build_orchestration_snapshot( + project: PyRef<'_, NativeProjectView>, + metadata_json: &str, + project_dir: PathBuf, + declarations: Option>>, +) -> PyResult { + let py = project.py(); + let mut input: SnapshotInput = serde_json::from_str(metadata_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !input.nodes.is_empty() { + return Err(PyValueError::new_err( + "snapshot nodes must come from the compiled project", + )); + } + if project + .effective_target_name() + .is_some_and(|target| target != input.target) + { + return Err(PyValueError::new_err( + "snapshot target differs from the compiled project", + )); + } + for diagnostic in project.diagnostics(py).bind(py).try_iter()? { + let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; + if severity == DiagnosticSeverity::Error.as_str() { + return Err(PyValueError::new_err( + "cannot publish a project with compiler errors", + )); + } + } + input.sqb_version = env!("CARGO_PKG_VERSION").into(); + input.environment_hash = environment_hash(py, &project_dir)?; + let owner = project.owner().bind(py).borrow(); + input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + drop(owner); + let declarations = match declarations { + Some(declarations) => declarations, + None => discover_declarations(py, &project_dir)?, + }; + if !declarations.is_empty() { + let authored = collect_declarations(py, declarations)?; + let graph = project.lineage_graph(py)?; + input = compile_declarations( + py, + input, + (&project_dir, graph.bind(py).get().graph()), + authored, + )?; + } + input.nodes = identity_nodes(project.into_pyobject(py)?.as_any(), input.nodes)?; + Ok(NativeOrchestrationSnapshot { input }) +} + +fn identity_nodes( + project: &Bound<'_, PyAny>, + mut nodes: BTreeMap, +) -> PyResult> { + let py = project.py(); + let options = PyDict::new(py); + options.set_item("project", project)?; + let scope = build_planner_scope(py, Some(options))?; + let identities = direct_model_version_identities( + (&field(project, "functions")?, &scope), + field(project, "sql_analysis_dialect")?.extract()?, + ( + &field(project, "seeds")?, + &py.None().into_bound(py), + &field(project, "hook_functions")?, + ), + )?; + for node in nodes.values_mut() { + let hashes = match node.kind.as_str() { + "model" => "model_local_hashes", + "udf" | "table_fn" => "function_local_hashes", + "seed" => "seed_version_hashes", + _ => continue, + }; + let name = node.definition["name"].as_str().unwrap_or_default(); + let hash: String = field(&identities, hashes)?.get_item(name)?.extract()?; + node.definition["planner_identity"] = json!({"local_hash": hash}); + } + Ok(nodes) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index c3169eea69..10c04fa2ef 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -201,6 +201,15 @@ class SnapshotCatalogTestCase: expected_kinds: frozenset[str] +@dataclass(frozen=True) +class SnapshotIdentityTestCase: + description: str + path: str + before: str + after: str + expected_changed: list[str] + + @dataclass(frozen=True) class SnapshotRejectionTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 56c4580a8c..0d7a1e564f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -34,6 +34,7 @@ SnapshotDiscoveryRejectionTestCase, SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, + SnapshotIdentityTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, ) @@ -199,6 +200,69 @@ def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_an assert publications[0] == publications[1] +@pytest.mark.parametrize( + "test_case", + [ + SnapshotIdentityTestCase(**case) + for case in json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json").read_text() + )["cases"] + ], + ids=lambda case: case.description, +) +def test_given_semantic_edits_when_publishing_then_local_identities_keep_planner_boundaries( + test_case: SnapshotIdentityTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json").read_text() + ) + files: dict[str, str] = ( + TYPE_PROOF_FILES + | fixture["files"] + | { + path: (Path("benchmarks/duckshop/project") / path).read_text() + for path in ( + "seeds/edge_cases/edge_quantity_steps.csv", + "seeds/edge_cases/edge_quantity_steps.yml", + "functions/sql/edge_cases/edge_next_quantity.sql", + ) + } + ) + original: str = files[test_case.path] + assert test_case.before in original + changed: dict[str, str] = files | { + test_case.path: original.replace(test_case.before, test_case.after) + } + identities: list[dict[str, str]] = [] + for label, inputs in (("original", files), ("changed", changed)): + directory: Path = tmp_path / label + write_files(project_dir=directory, files=inputs) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=directory), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + snapshot: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), directory).to_json() + ) + hashes: dict[str, str] = {} + for key in ( + "model:orders", + "model:stg_orders", + "udf:edge_next_quantity", + "seed:edge_quantity_steps", + ): + identity: dict[str, str] = snapshot["nodes"][key]["definition"]["planner_identity"] + assert set(identity) == {"local_hash"} + assert len(identity["local_hash"]) == 64 + hashes[key] = identity["local_hash"] + identities.append(hashes) + assert {key: identities[0][key] != identities[1][key] for key in identities[0]} == { + key: key in test_case.expected_changed for key in identities[0] + } + + @pytest.mark.parametrize( "test_case", [ From 12d51ae5a056c5f9e8e9c730f7f8bee873052646 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:21:42 +0000 Subject: [PATCH 103/138] feat: expose bounded node evidence and local definition status --- Cargo.lock | 2 + .../edge_cases/orchestration/node_status.json | 39 ++++ clients/typescript/src/schema.d.ts | 103 +++++++++ clients/typescript/tests/types.ts | 13 ++ .../_helpers/function_hashes.rs | 6 +- .../planner_changes/_helpers/json_payload.rs | 80 ------- .../src/planner_changes/_helpers/mod.rs | 1 - .../with_origin_cursor_input_names.rs | 4 +- .../src/planner_changes/constants.rs | 8 - .../main/recorded_declared_columns_hash.rs | 4 +- .../main/version_identity_payload.rs | 20 +- .../main/with_declared_columns_hash.rs | 6 +- .../main/without_declared_columns_hash.rs | 6 +- .../_helpers/compose_graph_identity.rs | 2 +- .../src/planner_identity/main/mod.rs | 2 - .../tests/test_planner_identity.rs | 4 +- .../main/stored_migration_fingerprint.rs | 2 +- crates/sqlbuild-core/Cargo.toml | 1 + .../src/fingerprint_identity/constants.rs | 6 + .../main/canonical_text.rs | 33 +++ .../main/decoded_object.rs | 12 + .../src/fingerprint_identity/main/mod.rs | 7 + .../main/model_local_identity_hash.rs | 2 +- .../normalize_version_identity_metadata.rs | 24 ++ .../fingerprint_identity/main/python_equal.rs | 40 ++++ .../main/recorded_model_local_hash.rs | 27 +++ .../main/stable_identity_hash.rs | 0 .../src/fingerprint_identity/mod.rs | 4 + crates/sqlbuild-core/src/lib.rs | 1 + crates/sqlbuild-orchestration/Cargo.toml | 1 + .../authentication/password_sessions.rs | 4 + .../_helpers/resources/person_operations.rs | 27 ++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/snapshots/_helpers/reading.rs | 24 +- .../src/snapshots/main/current_snapshot.rs | 3 +- .../snapshots/main/current_snapshot_record.rs | 2 +- .../src/snapshots/main/mod.rs | 1 + .../src/snapshots/main/read_current_record.rs | 22 ++ .../_helpers/fingerprint_reads.rs | 56 +++++ .../src/warehouse_metadata/_helpers/mod.rs | 2 + .../_helpers/node_status.rs | 75 ++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../main/read_person_fingerprint.rs | 40 +--- .../main/read_person_node_status.rs | 90 ++++++++ .../src/warehouse_metadata/models.rs | 47 ++++ .../tests/api/fingerprint_reads/helpers.rs | 3 +- .../tests/api/node_status/helpers.rs | 215 ++++++++++++++++++ .../tests/api/test_catalog_reads.rs | 24 +- .../tests/api/test_contracts.rs | 2 + .../_helpers/exported_constants/constants.rs | 6 +- .../_helpers/planner_identity/identities.rs | 2 +- .../_helpers/planner_identity/metadata.rs | 2 +- .../publication/person/helpers.py | 24 +- .../publication/test_person_journey.py | 11 +- 54 files changed, 954 insertions(+), 190 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_status.json delete mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/constants.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs rename crates/{sqlbuild-analysis/src/planner_identity => sqlbuild-core/src/fingerprint_identity}/main/model_local_identity_hash.rs (79%) create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs rename crates/{sqlbuild-analysis/src/planner_identity => sqlbuild-core/src/fingerprint_identity}/main/stable_identity_hash.rs (100%) create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index 34423dbe68..3876490943 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3683,6 +3683,7 @@ name = "sqlbuild-core" version = "0.187.3" dependencies = [ "regex", + "serde_json", "sha2 0.10.9", ] @@ -3744,6 +3745,7 @@ dependencies = [ "semver", "serde", "serde_json", + "sqlbuild-core", "sqlbuild-recall", "sqlbuild-state", "sqlx-core", diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_status.json b/benchmarks/duckshop/edge_cases/orchestration/node_status.json new file mode 100644 index 0000000000..420e0486be --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_status.json @@ -0,0 +1,39 @@ +{ + "model_metadata": { + "model_name": "orders", + "config": {"materialized": "table", "lookback": "2d"}, + "execution_signature": {"custom_config": {"label": "orders 🧾", "threshold": 1.25}}, + "local_function_hashes": {"order_total": "function-v1"} + }, + "model_local_hash": "34dd67f2ccccda42100a392d5bb611da80d22205a358cd5a45e8dc1c3f6d7e4b", + "query_hash": "orders-definition-latest", + "same_named_function": { + "name": "orders", + "destination": {"database": null, "schema": "main", "name": "orders"}, + "arguments": [], + "returns": "INTEGER", + "deps": [] + }, + "physical": { + "state_database": null, + "state_schema": "main", + "name": "orders", + "relation_type": "BASE TABLE", + "columns": [{"name": "order_id", "data_type": "INTEGER"}] + }, + "freshness": { + "event_id": "orders-source-newer", + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders", + "strategy": "max_column", + "value_kind": "integer", + "data_version": "18446744073709551619", + "data_version_hash": "orders-source-newer-version", + "observed_at": "2026-02-03T04:05:06Z" + } +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index fea1bb51af..fe88e0b2aa 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -195,6 +195,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/node-status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonNodeStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -807,6 +823,22 @@ export interface components { /** @enum {string} */ status: "timed_out"; }; + /** @description Local definition identity only, not upstream data or graph currency. */ + DefinitionEvidence: { + local_hash: string; + /** @enum {string} */ + status: "matched"; + } | { + current_local_hash: string; + recorded_local_hash: string; + /** @enum {string} */ + status: "changed"; + } | { + current_local_hash?: string | null; + reason: string; + /** @enum {string} */ + status: "unavailable"; + }; ExternalIdentityResponse: { csrf_token: string; /** Format: date-time */ @@ -1005,6 +1037,15 @@ export interface components { NodeEventKind: "success" | "failed" | "recovered"; /** @enum {string} */ NodeOutcome: "success" | "warn" | "failed" | "skipped" | "cancelled"; + NodeStatus: { + build?: null | components["schemas"]["FingerprintObservation"]; + definition: components["schemas"]["DefinitionEvidence"]; + node_key: string; + physical: components["schemas"]["PhysicalEvidence"]; + /** Format: uuid */ + snapshot_id: string; + source_data: components["schemas"]["SourceDataStatus"]; + }; OldNameViewsBatch: { /** Format: int64 */ generation: number; @@ -1073,6 +1114,16 @@ export interface components { csrf_token: string; identity: components["schemas"]["SessionIdentity"]; }; + /** @description Observed physical evidence is not a live warehouse-existence guarantee. */ + PhysicalEvidence: { + observation: components["schemas"]["CatalogObservation"]; + /** @enum {string} */ + status: "observed"; + } | { + reason: string; + /** @enum {string} */ + status: "unavailable"; + }; /** @enum {string} */ PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; /** @description One invocation's records; the server owns its claim namespace, start time and version. */ @@ -1431,6 +1482,11 @@ export interface components { snapshot_id: string; target: string; }; + SourceDataStatus: { + reason: string; + /** @enum {string} */ + status: "unknown"; + }; /** @description A bounded page from one registered reader of a published location/target generation. */ SourceFreshnessBatch: { /** Format: int64 */ @@ -2256,6 +2312,53 @@ export interface operations { }; }; }; + readPersonNodeStatus: { + parameters: { + query: { + /** @description One exact published node key */ + node_key: string; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["NodeStatus"]; + }; + }; + /** @description Invalid node key */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 07e2d4ee21..73c6560b93 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -122,3 +122,16 @@ void client.GET("/api/control/locations/{location_id}/targets/{target}/fingerpri }); // @ts-expect-error Build evidence does not contain SQL or arbitrary metadata. void ({} as components["schemas"]["FingerprintObservation"]).definition; +void client.GET("/api/control/locations/{location_id}/targets/{target}/node-status", { + params: { path: { location_id: "orders-location", target: "development" }, + query: { node_key: "model:orders" } }, +}); +const sourceData: components["schemas"]["SourceDataStatus"] = { + status: "unknown", reason: "consumed-input provenance is not recorded", +}; +void sourceData; +// @ts-expect-error Observation times cannot establish consumed-source currency. +const inferredCurrency: components["schemas"]["SourceDataStatus"] = { status: "behind", reason: "newer observation" }; +void inferredCurrency; +// @ts-expect-error Node status requires one exact published node. +void client.GET("/api/control/locations/{location_id}/targets/{target}/node-status", { params: { path: { location_id: "orders-location", target: "development" } } }); diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs index d3d966b00b..21ed40cb98 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs @@ -2,9 +2,11 @@ use serde_json::Value; -use crate::planner_changes::_helpers::json_payload::{decoded_object, python_equal}; -use crate::planner_changes::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY; use crate::planner_changes::models::IdentityPayload; +use sqlbuild_core::fingerprint_identity::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::{ + decoded_object::decoded_object, python_equal::python_equal, +}; /// Python equality of two identity payloads. pub(crate) fn payloads_equal(left: &IdentityPayload, right: &IdentityPayload) -> bool { diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs deleted file mode 100644 index 6b7e1ec92b..0000000000 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Decode fingerprint metadata as Python's `json.loads` does and write it as `json.dumps(sort_keys=True, separators=(",", ":"))`. - -use serde_json::{Map, Value}; -use sqlbuild_core::json::main::dumps::dumps; -use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; - -/// The decoded top-level object, or None when the text is not a JSON object. -pub(crate) fn decoded_object(metadata_json: &str) -> Result>, ()> { - match serde_json::from_str::(metadata_json) { - Ok(Value::Object(object)) => Ok(Some(object)), - Ok(_) => Ok(None), - Err(_) => Err(()), - } -} - -/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`. -pub(crate) fn canonical_text(object: Map) -> String { - let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { - sort_keys: true, - ..StdlibJsonOptions::new(None).with_separators(",", ":") - }); - dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default() -} - -/// Python equality of two decoded JSON values: numbers by value and booleans as 0 and 1. -pub(crate) fn python_equal(left: &Value, right: &Value) -> bool { - match (left, right) { - (Value::Array(left), Value::Array(right)) => { - left.len() == right.len() - && left - .iter() - .zip(right) - .all(|(left, right)| python_equal(left, right)) - } - (Value::Object(left), Value::Object(right)) => { - left.len() == right.len() - && left.iter().all(|(key, value)| { - right - .get(key) - .is_some_and(|other| python_equal(value, other)) - }) - } - (Value::Number(left_number), Value::Number(right_number)) - if !left_number.is_f64() && !right_number.is_f64() => - { - left_number == right_number - } - (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => { - numeric(left) == numeric(right) - } - _ => left == right, - } -} - -fn numeric(value: &Value) -> Option { - match value { - Value::Bool(flag) => Some(f64::from(u8::from(*flag))), - Value::Number(number) => number.as_f64(), - _ => None, - } -} - -fn json_value(value: Value) -> JsonValue { - match value { - Value::Null => JsonValue::Null, - Value::Bool(flag) => JsonValue::Bool(flag), - Value::Number(number) => match JsonInteger::parse(&number.to_string()) { - Some(integer) if !number.is_f64() => JsonValue::Integer(integer), - _ => JsonValue::Float(number.as_f64().unwrap_or_default()), - }, - Value::String(text) => JsonValue::String(text), - Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()), - Value::Object(object) => JsonValue::Object( - object - .into_iter() - .map(|(key, value)| (key, json_value(value))) - .collect(), - ), - } -} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs index f344a81462..093209583e 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod function_hashes; -pub(crate) mod json_payload; pub(crate) mod pick_more_aggressive; pub(crate) mod rename_mappings; pub(crate) mod schema_comparer; diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs index 77bd0cc2bc..ba76751400 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs @@ -2,8 +2,10 @@ use serde_json::{Map, Value}; -use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; use crate::planner_changes::constants::{CONFIG_METADATA_KEY, CURSOR_INPUTS_CONFIG_KEY}; +use sqlbuild_core::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; /// Canonical metadata JSON whose `config.cursor_inputs` keys use origin names; other text is returned unchanged. pub(crate) fn with_origin_cursor_input_names( diff --git a/crates/sqlbuild-analysis/src/planner_changes/constants.rs b/crates/sqlbuild-analysis/src/planner_changes/constants.rs index 0ff28f52b8..5bd2eae293 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/constants.rs @@ -1,13 +1,5 @@ //! Planner change-detection constants shared with the Python planner. -/// Fingerprint metadata key of the recorded audit gate. -pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; -/// Fingerprint metadata key of the migration fingerprint. -pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; -/// Fingerprint metadata key of the declared-columns hash. -pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; -/// Fingerprint metadata key of directly called function hashes. -pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; /// Fingerprint metadata key of the model config. pub const CONFIG_METADATA_KEY: &str = "config"; /// Model config key of cursor inputs. diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs index 6a3fa0c963..44c9a86876 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs @@ -2,8 +2,8 @@ use serde_json::Value; -use crate::planner_changes::_helpers::json_payload::decoded_object; -use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::decoded_object::decoded_object; /// The recorded declared-columns hash, when the metadata holds one as a string. pub fn recorded_declared_columns_hash(metadata_json: Option<&str>) -> Option { diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs index 9bceac24a7..62976bbb3b 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs @@ -1,11 +1,8 @@ //! The fingerprint metadata that participates in version identity. -use crate::planner_changes::_helpers::json_payload::decoded_object; -use crate::planner_changes::constants::{ - AUDIT_GATE_METADATA_KEY, DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, - MIGRATION_FINGERPRINT_METADATA_KEY, -}; use crate::planner_changes::models::IdentityPayload; +use sqlbuild_core::fingerprint_identity::main::decoded_object::decoded_object; +use sqlbuild_core::fingerprint_identity::main::normalize_version_identity_metadata::normalize_version_identity_metadata; /// The metadata without audit-gate, migration and declared-column entries; with `without_functions`, also without function hashes. pub fn version_identity_payload( @@ -16,19 +13,10 @@ pub fn version_identity_payload( return IdentityPayload::Absent; }; match decoded_object(text) { - Err(()) => IdentityPayload::Raw(text.to_owned()), + Err(_) => IdentityPayload::Raw(text.to_owned()), Ok(None) => IdentityPayload::Absent, Ok(Some(mut object)) => { - for key in [ - AUDIT_GATE_METADATA_KEY, - MIGRATION_FINGERPRINT_METADATA_KEY, - DECLARED_COLUMNS_METADATA_KEY, - ] { - object.shift_remove(key); - } - if without_functions { - object.shift_remove(LOCAL_FUNCTION_HASHES_METADATA_KEY); - } + normalize_version_identity_metadata(&mut object, without_functions); IdentityPayload::Object(object) } } diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs index c3508647a9..db131c7302 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs @@ -2,8 +2,10 @@ use serde_json::Value; -use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; -use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; /// Canonical metadata JSON that also records `declared_columns_hash`; text that is not an object is returned unchanged. pub fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs index a01da9c91e..01f2e9f785 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs @@ -1,7 +1,9 @@ //! Drop the declared-columns hash from fingerprint metadata. -use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; -use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; /// Canonical metadata JSON without the declared-columns hash, or the text unchanged when it has none. pub fn without_declared_columns_hash(metadata_json: &str) -> String { diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs index a8e54ec089..9850656bfc 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs @@ -1,6 +1,6 @@ //! A node identity composed from its local hash and the hashes of its available upstreams. -use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; /// The hash of the local hash and each upstream hash, one per line. pub(crate) fn compose_graph_identity(local_hash: &str, upstream_hashes: &[&str]) -> String { diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs index 94c535cd51..a7d42cd6e9 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs @@ -5,7 +5,5 @@ pub mod function_fingerprint_sql; pub mod graph_write_identity_hashes; pub mod missing_upstream_message; pub mod missing_upstreams; -pub mod model_local_identity_hash; -pub mod stable_identity_hash; pub mod stale_upstream_warning_message; pub mod topological_order; diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 4ce46945f6..6426f00d87 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -7,8 +7,6 @@ use crate::planner_identity::main::function_fingerprint_sql::function_fingerprin use crate::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; use crate::planner_identity::main::missing_upstream_message::missing_upstream_message; use crate::planner_identity::main::missing_upstreams::missing_upstreams; -use crate::planner_identity::main::model_local_identity_hash::model_local_identity_hash; -use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; use crate::planner_identity::main::topological_order::topological_order; use crate::planner_identity::models::{ @@ -25,6 +23,8 @@ use crate::planner_identity::tests::test_types::{ WarningTextTestCase, }; use crate::planner_identity::types::IdentityHashes; +use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; #[test] fn given_persisted_identity_inputs_when_hashing_then_matches_pinned_python_hashes() { diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs index c6f07fd0d5..34add163f5 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs @@ -1,6 +1,6 @@ //! Read persisted migration hashes without changing their representation. -use crate::planner_changes::constants::MIGRATION_FINGERPRINT_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::MIGRATION_FINGERPRINT_METADATA_KEY; /// Read only string migration fingerprints from object metadata. pub fn stored_migration_fingerprint(metadata: &str) -> Option { diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml index e70eb050a7..de423def04 100644 --- a/crates/sqlbuild-core/Cargo.toml +++ b/crates/sqlbuild-core/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] regex.workspace = true +serde_json.workspace = true sha2.workspace = true [lints] diff --git a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs new file mode 100644 index 0000000000..7b752a338a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs @@ -0,0 +1,6 @@ +//! Non-definition metadata excluded by the planner's version identity. + +pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; +pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; +pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; +pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs new file mode 100644 index 0000000000..4434934b12 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs @@ -0,0 +1,33 @@ +//! Python-compatible canonical JSON for planner fingerprint metadata. + +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; +use serde_json::{Map, Value}; + +/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`. +pub fn canonical_text(object: Map) -> String { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }); + dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default() +} + +fn json_value(value: Value) -> JsonValue { + match value { + Value::Null => JsonValue::Null, + Value::Bool(flag) => JsonValue::Bool(flag), + Value::Number(number) => match JsonInteger::parse(&number.to_string()) { + Some(integer) if !number.is_f64() => JsonValue::Integer(integer), + _ => JsonValue::Float(number.as_f64().unwrap_or_default()), + }, + Value::String(text) => JsonValue::String(text), + Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()), + Value::Object(object) => JsonValue::Object( + object + .into_iter() + .map(|(key, value)| (key, json_value(value))) + .collect(), + ), + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs new file mode 100644 index 0000000000..ac1d41eb1b --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs @@ -0,0 +1,12 @@ +//! Decode fingerprint metadata as Python's JSON object reader does. + +use serde_json::{Map, Value}; + +pub fn decoded_object( + metadata_json: &str, +) -> Result>, serde_json::Error> { + Ok(match serde_json::from_str::(metadata_json)? { + Value::Object(object) => Some(object), + _ => None, + }) +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs new file mode 100644 index 0000000000..914d2d4cf4 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs @@ -0,0 +1,7 @@ +pub mod canonical_text; +pub mod decoded_object; +pub mod model_local_identity_hash; +pub mod normalize_version_identity_metadata; +pub mod python_equal; +pub mod recorded_model_local_hash; +pub mod stable_identity_hash; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/model_local_identity_hash.rs similarity index 79% rename from crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs rename to crates/sqlbuild-core/src/fingerprint_identity/main/model_local_identity_hash.rs index 11148c52dc..585c7ada95 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/model_local_identity_hash.rs @@ -1,6 +1,6 @@ //! A model's local identity hash from its query fingerprint and non-query metadata. -use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; +use crate::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; /// The hash of the query fingerprint and metadata JSON, one per line. pub fn model_local_identity_hash(query_fingerprint: &str, metadata_json: &str) -> String { diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs new file mode 100644 index 0000000000..f84f44064a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs @@ -0,0 +1,24 @@ +//! Normalize recorded metadata without changing the persisted fingerprint. + +use serde_json::{Map, Value}; + +use crate::fingerprint_identity::constants::{ + AUDIT_GATE_METADATA_KEY, DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, +}; + +pub fn normalize_version_identity_metadata( + object: &mut Map, + without_functions: bool, +) { + for key in [ + AUDIT_GATE_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, + DECLARED_COLUMNS_METADATA_KEY, + ] { + object.shift_remove(key); + } + if without_functions { + object.shift_remove(LOCAL_FUNCTION_HASHES_METADATA_KEY); + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs new file mode 100644 index 0000000000..6feffc311d --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs @@ -0,0 +1,40 @@ +//! Python equality of decoded fingerprint values, including numeric booleans. + +use serde_json::Value; + +pub fn python_equal(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Array(left), Value::Array(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| python_equal(left, right)) + } + (Value::Object(left), Value::Object(right)) => { + left.len() == right.len() + && left.iter().all(|(key, value)| { + right + .get(key) + .is_some_and(|other| python_equal(value, other)) + }) + } + (Value::Number(left_number), Value::Number(right_number)) + if !left_number.is_f64() && !right_number.is_f64() => + { + left_number == right_number + } + (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => { + numeric(left) == numeric(right) + } + _ => left == right, + } +} + +fn numeric(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + _ => None, + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs new file mode 100644 index 0000000000..bb91cdbb3a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs @@ -0,0 +1,27 @@ +//! Recover model-local identity only from a complete recorded planner payload. + +use crate::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use crate::fingerprint_identity::main::normalize_version_identity_metadata::normalize_version_identity_metadata; +use crate::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; + +pub fn recorded_model_local_hash(query_hash: &str, metadata_json: &str) -> Option { + let mut object = match decoded_object(metadata_json) { + Ok(Some(object)) => object, + Ok(None) | Err(_) => return None, + }; + if query_hash.is_empty() + || !object.get("model_name")?.is_string() + || !object.get("config")?.is_object() + || !object.get("execution_signature")?.is_object() + || !object.get("local_function_hashes")?.is_object() + { + return None; + } + normalize_version_identity_metadata(&mut object, false); + Some(model_local_identity_hash( + query_hash, + &canonical_text(object), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/stable_identity_hash.rs similarity index 100% rename from crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs rename to crates/sqlbuild-core/src/fingerprint_identity/main/stable_identity_hash.rs diff --git a/crates/sqlbuild-core/src/fingerprint_identity/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs new file mode 100644 index 0000000000..919648bfd1 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs @@ -0,0 +1,4 @@ +//! Pure planner fingerprint identity, shared by planning and stored-evidence reads. + +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 29ad9c99e3..0b6f781eea 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -28,6 +28,7 @@ pub mod diagnostic_routing; pub mod diagnostic_text; pub mod dlt_destination; pub mod event_export; +pub mod fingerprint_identity; pub mod format_command; pub mod freshness_text; pub mod git_revisions; diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml index fbddfad20f..6c2b733287 100644 --- a/crates/sqlbuild-orchestration/Cargo.toml +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -23,6 +23,7 @@ rusqlite.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true +sqlbuild-core.workspace = true sqlbuild-recall.workspace = true sqlbuild-state.workspace = true sqlx-core.workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 54051fd839..9bc2ab2128 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -70,6 +70,10 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/control/locations/{location_id}/targets/{target}/fingerprint", get(crate::api::_helpers::resources::person_operations::fingerprint), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/node-status", + get(crate::api::_helpers::resources::person_operations::node_status), + ) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 0de02072dc..32708a27c7 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -19,8 +19,10 @@ use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunReco use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; +use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; use crate::warehouse_metadata::models::{ - CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, + CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, + NodeStatusQuery, }; #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", @@ -164,3 +166,26 @@ pub(crate) async fn fingerprint( .await?, )) } + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/node-status", operation_id = "readPersonNodeStatus", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("node_key" = String, Query, description = "One exact published node key")), + responses((status = 200, body = Option), (status = 400, description = "Invalid node key"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn node_status( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + headers: HeaderMap, +) -> Result>, ApiError> { + Ok(Json( + read_person_node_status( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 0caf4e5d7d..c5d71e4f3e 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -22,6 +22,7 @@ use crate::api::_helpers::resources::{ person_operations::recent_runs, person_operations::catalog, person_operations::fingerprint, + person_operations::node_status, password_sessions::login, password_sessions::current, password_sessions::logout, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs index 7f463dd024..e96bb1fec3 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -2,30 +2,12 @@ use std::collections::BTreeMap; -use serde::de::DeserializeOwned; -use serde_json::Value; -use uuid::Uuid; - -use crate::snapshots::models::{SnapshotNode, SnapshotRecord}; +use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; use crate::store::main::decompress::decompress; -use crate::store::main::{decode_row::decode_row, query::query}; -use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; - -pub(crate) async fn read_current_record( - transaction: ControlTransaction, - (location_id, target): (Uuid, &str), -) -> Result<(ControlTransaction, Option), ControlError> { - let (transaction, rows) = query(transaction, - "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ - FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", - vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; - Ok(( - transaction, - rows.into_iter().next().map(decode_row).transpose()?, - )) -} +use serde::de::DeserializeOwned; +use serde_json::Value; fn decode_body(value: &Value) -> Result { serde_json::from_slice(&decompress(value)?) diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs index 81889f713b..ddffa096f8 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -2,7 +2,8 @@ use uuid::Uuid; -use crate::snapshots::_helpers::reading::{attach_sources, read_current_record}; +use crate::snapshots::_helpers::reading::attach_sources; +use crate::snapshots::main::read_current_record::read_current_record; use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::{CurrentSnapshot, SnapshotChange}; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs index 58f44ee9a4..940c0a3804 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::snapshots::_helpers::reading::read_current_record; +use crate::snapshots::main::read_current_record::read_current_record; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index f89796c34a..ba10af4391 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -5,6 +5,7 @@ pub mod next_remote_generation; pub(crate) mod publish_development_snapshot; pub mod publish_remote_snapshot; pub mod publish_snapshot; +pub(crate) mod read_current_record; pub(crate) mod read_snapshot_documents; pub(crate) mod read_snapshot_nodes; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs new file mode 100644 index 0000000000..a5be66e12f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs @@ -0,0 +1,22 @@ +//! Read publication identity inside the caller's existing authorization transaction. + +use uuid::Uuid; + +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_current_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs new file mode 100644 index 0000000000..dd3bd789e8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs @@ -0,0 +1,56 @@ +//! One indexed exact-node/destination read, shared by evidence and definition comparison. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; + +pub(crate) async fn latest_fingerprint( + transaction: ControlTransaction, + (location_id, target, options): (Uuid, &str, &FingerprintQuery), +) -> Result<(ControlTransaction, Option<(FingerprintObservation, String)>), ControlError> { + if options.node_type.trim().is_empty() || options.node_name.trim().is_empty() { + return Err(ControlError::Invalid( + "fingerprint reads require an exact node".into(), + )); + } + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.node_type.clone()), + Parameter::Text(options.node_name.clone()), + ]; + let mut predicates: Vec = Vec::new(); + for (column, value) in [ + ("target_database", &options.database), + ("target_schema", &options.schema), + ("target_name", &options.name), + ] { + predicates.push(if let Some(value) = value { + parameters.push(Parameter::Text(value.clone())); + format!("{column}=${}", parameters.len()) + } else { + format!("{column} IS NULL") + }); + } + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,observed_at,source_event_id,record_json ->> 'run_id' AS run_id, \ + record_json ->> 'definition_hash' AS definition_hash,record_json ->> 'version_hash' AS version_hash, \ + record_json ->> 'schema_fingerprint' AS schema_fingerprint,record_json ->> 'metadata_json' AS metadata_json FROM sqb_warehouse_fingerprints \ + WHERE location_id=$1 AND target=$2 AND node_type=$3 AND node_name=$4 AND {} \ + ORDER BY target_database,target_schema,target_name,observed_at DESC,event_id DESC LIMIT 1", predicates.join(" AND ")), parameters).await?; + let evidence = rows + .into_iter() + .next() + .map(|mut row| { + let metadata = row + .remove("metadata_json") + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_default(); + Ok::<_, ControlError>((decode_row(row)?, metadata)) + }) + .transpose()?; + Ok((transaction, evidence)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs index e3204dbb07..b5049128dc 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -1,7 +1,9 @@ pub(crate) mod catalog; pub(crate) mod events; +pub(crate) mod fingerprint_reads; pub(crate) mod fingerprints; pub(crate) mod freshness; pub(crate) mod ingestion; +pub(crate) mod node_status; pub(crate) mod person_reads; pub(crate) mod reconciliation; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs new file mode 100644 index 0000000000..57a1ede606 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs @@ -0,0 +1,75 @@ +//! Physical observations and planner-local definitions remain independent evidence dimensions. + +use sqlbuild_core::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; +use uuid::Uuid; + +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::scoped_event_id; +use crate::warehouse_metadata::models::{ + DefinitionEvidence, FingerprintObservation, FingerprintQuery, PhysicalEvidence, +}; + +pub(crate) fn destination(node: &SnapshotNode) -> Option { + let destination = node.definition.get("destination")?; + Some(FingerprintQuery { + node_type: node.kind.clone(), + node_name: node.definition["name"].as_str()?.into(), + database: destination["database"].as_str().map(str::to_owned), + schema: Some(destination["schema"].as_str()?.into()), + name: Some(destination["name"].as_str()?.into()), + }) +} + +pub(crate) fn definition_evidence( + node: &SnapshotNode, + build: Option<&(FingerprintObservation, String)>, +) -> DefinitionEvidence { + let current = node.definition["planner_identity"]["local_hash"].as_str(); + let recorded = build.and_then(|(record, metadata)| match node.kind.as_str() { + "model" => recorded_model_local_hash(&record.definition_hash, metadata), + _ => None, + }); + match (current, recorded) { + (Some(current), Some(recorded)) if current == recorded => DefinitionEvidence::Matched { + local_hash: recorded, + }, + (Some(current), Some(recorded)) => DefinitionEvidence::Changed { + current_local_hash: current.into(), + recorded_local_hash: recorded, + }, + _ => DefinitionEvidence::Unavailable { + reason: "comparable planner-local model identities are unavailable".into(), + current_local_hash: current.map(str::to_owned), + }, + } +} + +pub(crate) async fn physical_evidence( + transaction: ControlTransaction, + (location_id, target, destination): (Uuid, &str, &FingerprintQuery), +) -> Result<(ControlTransaction, PhysicalEvidence), ControlError> { + let relation_id = scoped_event_id( + (location_id, target), + ( + destination.database.as_deref(), + destination.schema.as_deref().unwrap_or_default(), + "catalog", + ), + destination.name.as_deref().unwrap_or_default(), + )?; + let (transaction, rows) = query(transaction, + "SELECT event_id,observed_at,record_json AS record FROM sqb_warehouse_catalog WHERE event_id=$1 AND location_id=$2 AND target=$3 LIMIT 1", + vec![Parameter::Id(relation_id), Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let evidence = match rows.into_iter().next() { + Some(row) => PhysicalEvidence::Observed { + observation: decode_row(row)?, + }, + None => PhysicalEvidence::Unavailable { + reason: "no retained physical catalog observation for the exact destination".into(), + }, + }; + Ok((transaction, evidence)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index f3355f9c82..3060cb58f4 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -9,4 +9,5 @@ pub mod ingest_old_name_views; pub mod ingest_source_freshness; pub mod read_person_catalog; pub mod read_person_fingerprint; +pub mod read_person_node_status; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs index 2d2ca1660d..bbb36ad172 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs @@ -4,8 +4,9 @@ use uuid::Uuid; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::store::errors::ControlError; -use crate::store::main::{commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{ControlStore, Parameter}; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprint; use crate::warehouse_metadata::_helpers::person_reads::begin_view; use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; @@ -27,37 +28,8 @@ pub async fn read_person_fingerprint( default_role, ) .await?; - if options.node_type.trim().is_empty() || options.node_name.trim().is_empty() { - return Err(ControlError::Invalid( - "fingerprint reads require an exact node".into(), - )); - } - let mut parameters = vec![ - Parameter::Id(location_id), - Parameter::Text(target.into()), - Parameter::Text(options.node_type.clone()), - Parameter::Text(options.node_name.clone()), - ]; - let mut predicates: Vec = Vec::new(); - for (column, value) in [ - ("target_database", &options.database), - ("target_schema", &options.schema), - ("target_name", &options.name), - ] { - predicates.push(if let Some(value) = value { - parameters.push(Parameter::Text(value.clone())); - format!("{column}=${}", parameters.len()) - } else { - format!("{column} IS NULL") - }); - } - let (transaction, rows) = query(transaction, - &format!("SELECT event_id,observed_at,source_event_id,record_json ->> 'run_id' AS run_id, \ - record_json ->> 'definition_hash' AS definition_hash,record_json ->> 'version_hash' AS version_hash, \ - record_json ->> 'schema_fingerprint' AS schema_fingerprint FROM sqb_warehouse_fingerprints \ - WHERE location_id=$1 AND target=$2 AND node_type=$3 AND node_name=$4 AND {} \ - ORDER BY target_database,target_schema,target_name,observed_at DESC,event_id DESC LIMIT 1", predicates.join(" AND ")), parameters).await?; - let record = rows.into_iter().next().map(decode_row).transpose()?; + let (transaction, evidence) = + latest_fingerprint(transaction, (location_id, target, options)).await?; commit(transaction).await?; - Ok(record) + Ok(evidence.map(|(record, _)| record)) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs new file mode 100644 index 0000000000..f54bbdf0f2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs @@ -0,0 +1,90 @@ +//! One authorized current node, one physical observation and one latest build; never warehouse polling. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprint; +use crate::warehouse_metadata::_helpers::node_status::{ + definition_evidence, destination, physical_evidence, +}; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::models::{ + NodeStatus, NodeStatusQuery, PhysicalEvidence, SourceDataStatus, +}; + +pub async fn read_person_node_status( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location_id, target, options): (Uuid, &str, &NodeStatusQuery), + default_role: DefaultRole, +) -> Result, ControlError> { + let transaction = begin_view( + store, + credentials, + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [options.node_key.clone()].into(), + }, + default_role, + ) + .await?; + if options.node_key.trim().is_empty() { + return Err(ControlError::Invalid( + "node status requires an exact node key".into(), + )); + } + let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; + let Some(snapshot) = snapshot else { + commit(transaction).await?; + return Ok(None); + }; + let (mut transaction, mut nodes) = read_snapshot_nodes( + transaction, + snapshot.snapshot_id, + Some(&[options.node_key.clone()].into()), + ) + .await?; + let Some(node) = nodes.remove(&options.node_key) else { + commit(transaction).await?; + return Ok(None); + }; + let mut build = None; + let physical = if let Some(destination) = destination(&node) { + let physical = if matches!(node.kind.as_str(), "model" | "seed") { + let physical; + (transaction, physical) = + physical_evidence(transaction, (location_id, target, &destination)).await?; + physical + } else { + PhysicalEvidence::Unavailable { + reason: "node is not a physical relation".into(), + } + }; + (transaction, build) = + latest_fingerprint(transaction, (location_id, target, &destination)).await?; + physical + } else { + PhysicalEvidence::Unavailable { + reason: "node has no physical relation destination".into(), + } + }; + let status = NodeStatus { + snapshot_id: snapshot.snapshot_id, + node_key: options.node_key.clone(), + physical, + definition: definition_evidence(&node, build.as_ref()), + build: build.map(|(record, _)| record), + source_data: SourceDataStatus::Unknown { + reason: "consumed-input provenance is not recorded".into(), + }, + }; + commit(transaction).await?; + Ok(Some(status)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 8cfc89d70c..bdc99b9935 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -571,3 +571,50 @@ pub struct FingerprintObservation { pub version_hash: String, pub schema_fingerprint: String, } + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeStatusQuery { + pub node_key: String, +} + +/// Observed physical evidence is not a live warehouse-existence guarantee. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum PhysicalEvidence { + Observed { observation: CatalogObservation }, + Unavailable { reason: String }, +} + +/// Local definition identity only, not upstream data or graph currency. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum DefinitionEvidence { + Matched { + local_hash: String, + }, + Changed { + current_local_hash: String, + recorded_local_hash: String, + }, + Unavailable { + reason: String, + current_local_hash: Option, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum SourceDataStatus { + Unknown { reason: String }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct NodeStatus { + pub snapshot_id: Uuid, + pub node_key: String, + pub physical: PhysicalEvidence, + pub definition: DefinitionEvidence, + pub build: Option, + pub source_data: SourceDataStatus, +} diff --git a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs index 448710fae3..6eb59e6b87 100644 --- a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs @@ -19,7 +19,7 @@ pub(crate) async fn fingerprint_reads( store: &ControlStore, credentials: (&AuditContext, (&str, &str)), (location, other): (Uuid, Uuid), -) { +) -> Uuid { let session_id = Uuid::now_v7(); report_location_health( store, @@ -164,4 +164,5 @@ pub(crate) async fn fingerprint_reads( ), serialized ); + session_id } diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs new file mode 100644 index 0000000000..17cf8e75ae --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -0,0 +1,215 @@ +//! Real stored evidence never substitutes observation clocks for consumed inputs. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::{ + ingest_catalog::ingest_catalog, ingest_fingerprints::ingest_fingerprints, + ingest_source_freshness::ingest_source_freshness, + read_person_node_status::read_person_node_status, +}; +use crate::warehouse_metadata::models::{ + CatalogBatch, FingerprintBatch, NodeStatusQuery, SourceFreshnessBatch, +}; + +pub(crate) async fn node_status_reads( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location, session_id): (Uuid, Uuid), +) { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration"); + let data: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("node_status.json")).expect("fixture"), + ) + .expect("status inputs"); + let mut input = snapshot(location); + input.generation = 2; + input.nodes.insert("model:orders".into(), SnapshotNode { + kind: "model".into(), + definition: json!({"name":"orders","destination":{"database":null,"schema":"main","name":"orders"}, + "planner_identity":{"local_hash":data["model_local_hash"]},"deps":["source:raw_orders"]}), + raw_sql: None, compiled_sql: None, python_source: None, + }); + input.nodes.insert( + "udf:orders".into(), + SnapshotNode { + kind: "udf".into(), + definition: data["same_named_function"].clone(), + raw_sql: None, + compiled_sql: Some("23".into()), + python_source: None, + }, + ); + publish_snapshot(store, &input) + .await + .expect("identity snapshot"); + let options = NodeStatusQuery { + node_key: "model:orders".into(), + }; + let initial = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("legacy build") + ); + assert_eq!(initial["definition"]["status"], "unavailable"); + assert_eq!(initial["physical"]["status"], "unavailable"); + let fixture: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("fingerprint_lookup.json")).expect("build fixture"), + ) + .expect("build inputs"); + let mut record = fixture["record"].clone(); + let mut metadata = data["model_metadata"].clone(); + metadata["audit_gate"] = json!({"run_id":"audit-run","outcome":"passed"}); + metadata["migration_fingerprint"] = json!("migration-v1"); + metadata["declared_columns_hash"] = json!("columns-v1"); + record["metadata_json"] = json!(metadata.to_string()); + record["event_id"] = json!("orders-model-identity"); + record["ts"] = json!("2026-01-05T10:00:00Z"); + let batch: FingerprintBatch = + serde_json::from_value(json!({"session_id":session_id,"generation":2,"records":[record]})) + .expect("model build"); + ingest_fingerprints(store, (location, "development"), &batch) + .await + .expect("build ingestion"); + let before = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("matched definition without catalog") + ); + assert_eq!( + before["definition"], + json!({"status":"matched","local_hash":data["model_local_hash"]}) + ); + assert_eq!(before["physical"]["status"], "unavailable"); + let catalog: CatalogBatch = serde_json::from_value(json!({"session_id":session_id,"generation":2,"records":[data["physical"]], + "reconciliation":{"scan_id":Uuid::now_v7(),"page":0,"state_database":null,"state_schema":"main","complete":false,"day":null,"after":null}})).expect("catalog page"); + ingest_catalog(store, (location, "development"), &catalog) + .await + .expect("physical evidence"); + let freshness: SourceFreshnessBatch = serde_json::from_value( + json!({"session_id":session_id,"generation":2,"records":[data["freshness"]]}), + ) + .expect("newer source observation"); + assert!(freshness.records[0].observed_at > batch.records[0].ts); + ingest_source_freshness(store, (location, "development"), &freshness) + .await + .expect("source observation"); + let after = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("independent evidence") + ); + assert_eq!(after["physical"]["status"], "observed"); + assert_eq!(after["physical"]["observation"]["record"], data["physical"]); + assert_eq!(after["definition"], before["definition"]); + assert_eq!(after["source_data"], before["source_data"]); + assert_eq!( + after["source_data"], + json!({"status":"unknown","reason":"consumed-input provenance is not recorded"}) + ); + let mut function_batch = batch.clone(); + function_batch.records[0].node_type = "udf".into(); + function_batch.records[0].event_id = Some("orders-function-build".into()); + function_batch.records[0].run_id = "orders-function-run".into(); + ingest_fingerprints(store, (location, "development"), &function_batch) + .await + .expect("same-named function build"); + let function_status = json!( + read_person_node_status( + store, + credentials, + ( + location, + "development", + &NodeStatusQuery { + node_key: "udf:orders".into() + } + ), + DefaultRole::None + ) + .await + .expect("function namespace") + ); + assert_eq!(function_status["physical"]["status"], "unavailable"); + assert_eq!(function_status["definition"]["status"], "unavailable"); + assert_eq!(function_status["build"]["run_id"], "orders-function-run"); + assert_eq!(function_status["source_data"], after["source_data"]); + let mut changed = batch.clone(); + metadata["config"]["lookback"] = json!("7d"); + changed.records[0].metadata_json = metadata.to_string(); + changed.records[0].event_id = Some("orders-config-changed".into()); + changed.records[0].ts += chrono::Duration::days(1); + ingest_fingerprints(store, (location, "development"), &changed) + .await + .expect("config-only build"); + let changed_status = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("config comparison") + ); + assert_eq!(changed_status["definition"]["status"], "changed"); + assert_eq!( + changed_status["definition"]["current_local_hash"], + data["model_local_hash"] + ); + assert_ne!( + changed_status["definition"]["recorded_local_hash"], + data["model_local_hash"] + ); + assert_eq!(changed_status["source_data"], after["source_data"]); + assert!(!changed_status.to_string().contains("metadata_json")); + assert!(matches!( + read_person_node_status( + store, + credentials, + (location, "preview", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + read_person_node_status( + store, + credentials, + ( + location, + "development", + &NodeStatusQuery { + node_key: "model:unlisted".into() + } + ), + DefaultRole::None + ) + .await + .expect("unknown node") + .is_none() + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index ddab9dc6de..c83b56f94c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -174,7 +174,7 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp ) .await .expect("scoped viewer"); - crate::api::tests::fingerprint_reads_helpers::fingerprint_reads( + let reader_session = crate::api::tests::fingerprint_reads_helpers::fingerprint_reads( store, credentials, (location, other), @@ -288,6 +288,12 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp .await, Err(ControlError::Invalid(_)) )); + crate::api::tests::node_status_helpers::node_status_reads( + store, + credentials, + (location, reader_session), + ) + .await; assignment.revoke = true; assign_builtin_role( store, @@ -297,6 +303,22 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp ) .await .expect("revoke"); + assert!(matches!( + crate::warehouse_metadata::main::read_person_node_status::read_person_node_status( + store, + credentials, + ( + location, + "development", + &crate::warehouse_metadata::models::NodeStatusQuery { + node_key: "model:orders".into() + } + ), + DefaultRole::None, + ) + .await, + Err(ControlError::Denied { .. }) + )); let evidence = crate::warehouse_metadata::models::FingerprintQuery { node_type: "model".into(), node_name: "orders".into(), diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 7736c079fd..3e5744e2ff 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -6,6 +6,8 @@ mod changes_helpers; mod fingerprint_reads_helpers; #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "node_status/helpers.rs"] +mod node_status_helpers; #[path = "oidc/helpers.rs"] mod oidc_helpers; #[path = "oidc/test_types.rs"] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 6a71912e04..511a468e3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -8,9 +8,6 @@ use sqlbuild_analysis::fingerprint_storage::constants::{ COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; -use sqlbuild_analysis::planner_changes::constants::{ - AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, -}; use sqlbuild_analysis::snowflake_cost::constants::QUERY_TAG_APP; use sqlbuild_cli::compute_logs::constants::{ COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, @@ -29,6 +26,9 @@ use sqlbuild_core::constants::{ use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, }; +use sqlbuild_core::fingerprint_identity::constants::{ + AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, +}; use sqlbuild_host::entries::constants::CUSTOM_HOST_LAUNCH_MODULE; use sqlbuild_rules::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use sqlbuild_rules::custom_host_pool::constants::CUSTOM_RULE_PROJECT_SUBJECT; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index bf4b2868d7..cc372a504b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -11,8 +11,8 @@ use sqlbuild_analysis::planner_identity::constants::{ }; use sqlbuild_analysis::planner_identity::main::expected_graph_identity_hashes::expected_graph_identity_hashes; use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; -use sqlbuild_analysis::planner_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; +use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index f4d132463e..bc63fdbe52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -11,8 +11,8 @@ use sqlbuild_analysis::planner_identity::constants::{ VERSION_IDENTITY_CONFIG_KEYS, }; use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; -use sqlbuild_analysis::planner_identity::main::stable_identity_hash::stable_identity_hash; use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; use crate::bindings::_helpers::boundary::python_calls::{imported, keywords, within}; use crate::bindings::_helpers::compiled_views::function_reads; diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 8d1f8466f0..c90f2b8462 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -62,7 +62,12 @@ def person_read(api: SnapshotApi, path: str, *, session: tuple[str, str]) -> Any def wait_for_person_catalog( - api: SnapshotApi, session: tuple[str, str], process: Popen[bytes], log: Path + api: SnapshotApi, + session: tuple[str, str], + process: Popen[bytes], + log: Path, + *, + expected_definition_hash: str, ) -> None: """Wait for an actual worker-built relation through the scoped browser API, not store SQL.""" path: str = f"/api/control/locations/{api.location_id}/targets/development/catalog?schema=main" @@ -93,9 +98,9 @@ def wait_for_person_catalog( "?node_type=model&node_name=orders&schema=main&name=orders" ) evidence: dict[str, Any] | None = person_read(api, fingerprint_path, session=session) - while evidence is None: + while evidence is None or evidence["definition_hash"] != expected_definition_hash: assert process.poll() is None, log.read_text() - assert time.monotonic() < deadline, log.read_text() + assert time.monotonic() < deadline, (evidence, log.read_text()) time.sleep(0.05) evidence = person_read(api, fingerprint_path, session=session) assert set(evidence) == { @@ -109,6 +114,19 @@ def wait_for_person_catalog( } assert all(evidence.values()), evidence assert person_read(api, fingerprint_path + "&database=unlisted", session=session) is None + status: dict[str, Any] = person_read( + api, + f"/api/control/locations/{api.location_id}/targets/development/node-status?node_key=model:orders", + session=session, + ) + assert status["physical"]["status"] == "observed" + assert status["physical"]["observation"] == records["orders"] + assert status["definition"]["status"] == "matched", status + assert status["build"] == evidence + assert status["source_data"] == { + "status": "unknown", + "reason": "consumed-input provenance is not recorded", + } def verify_security_audit( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index 683fda72d1..f12446d360 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -12,6 +12,7 @@ import duckdb import pytest +from sqlbuild._native import compute_query_hash from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( PersonJourneyTestCase, WorkerExecutionTestCase, @@ -239,7 +240,15 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi str(worker.project / "warehouse.duckdb"), read_only=True ) as warehouse: assert warehouse.sql(test_case.warehouse_query).fetchall() == [(41,)] - wait_for_person_catalog(worker.api, session, location, log) + wait_for_person_catalog( + worker.api, + session, + location, + log, + expected_definition_hash=compute_query_hash( + query_sql=test_case.model_sql.replace("23", "41"), dialect="duckdb" + ), + ) assignment["revoke"] = True assert person_request( worker.api, "/api/admin/roles", body=assignment, session=admin_session From bc188288e9153633fca1cbed52fe54bb1e8d9fa6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:57:46 +0000 Subject: [PATCH 104/138] feat: add metadata-only relation statistics estimates --- .../orchestration/warehouse_statistics.json | 10 ++ .../adapter/contract/classes/base_adapter.py | 1 + src/sqlbuild/adapter/contract/classes/diff.py | 3 +- .../contract/classes/duckdb_backed_adapter.py | 1 + src/sqlbuild/adapter/contract/models.py | 4 +- .../bigquery/classes/bigquery_adapter.py | 13 ++- .../clickhouse/classes/clickhouse_adapter.py | 26 +++-- .../databricks/classes/databricks_adapter.py | 20 +++- .../postgres/classes/postgres_adapter.py | 13 ++- .../snowflake/classes/snowflake_adapter.py | 13 ++- .../adapters/sqlite/classes/sqlite_adapter.py | 7 +- .../sqlserver/classes/sqlserver_adapter.py | 1 + .../sqlbuild/adapters/sqlite/_test_types.py | 7 ++ .../sqlbuild/adapters/sqlite/test_client.py | 49 ++++++++ .../unit/src/sqlbuild/adapters/_test_types.py | 6 + tests/unit/src/sqlbuild/adapters/helpers.py | 1 + .../test_relation_row_count_estimate.py | 108 +++++++++++++++++- 17 files changed, 255 insertions(+), 28 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json new file mode 100644 index 0000000000..0926e6a039 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json @@ -0,0 +1,10 @@ +{ + "sqlite": { + "setup_sql": [ + "CREATE TABLE orders (order_id INTEGER)", + "INSERT INTO orders VALUES (17), (31), (59)" + ], + "name": "orders", + "metadata_detail": "SQLite does not report table row counts from metadata" + } +} diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 6acc8921ad..683da84321 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -1661,6 +1661,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Report an unknown row count for custom adapters without metadata support.""" diff --git a/src/sqlbuild/adapter/contract/classes/diff.py b/src/sqlbuild/adapter/contract/classes/diff.py index ccb8b2078a..376cc6debe 100644 --- a/src/sqlbuild/adapter/contract/classes/diff.py +++ b/src/sqlbuild/adapter/contract/classes/diff.py @@ -84,8 +84,9 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: - """Return the metadata row count for one relation without scanning its data.""" + """Estimate counts, forbidding data scans when metadata_only is requested.""" ... @abstractmethod diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index fb8d0e8fe5..119592a31a 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -2270,6 +2270,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read DuckDB's estimated table size from the catalog.""" diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 5c0a97c92a..33862792c6 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -369,10 +369,12 @@ class RowDiffCoverage: @dataclass(frozen=True) class RelationRowCountEstimate: - """Metadata row count read without scanning data; ``row_count`` is None when unknown.""" + """Relation estimates; unavailable counts and storage bytes remain None.""" row_count: int | None detail: str | None = None + byte_count: int | None = None + byte_count_basis: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 1247bbc73c..7c6500412d 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -2569,6 +2569,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read ``numRows`` from the tables API; only native tables report a count.""" @@ -2579,11 +2580,19 @@ def estimate_relation_row_count( ) table_type: str = str(getattr(table, "table_type", NATIVE_TABLE_TYPE)).upper() num_rows: object | None = getattr(table, "num_rows", None) - if table_type != NATIVE_TABLE_TYPE or not isinstance(num_rows, int): + if table_type != NATIVE_TABLE_TYPE: return RelationRowCountEstimate( row_count=None, detail=f"no row count ({table_type.lower()})" ) - return RelationRowCountEstimate(row_count=num_rows) + byte_count: object | None = getattr(table, "num_bytes", None) if metadata_only else None + return RelationRowCountEstimate( + row_count=num_rows if isinstance(num_rows, int) else None, + detail=f"no row count ({table_type.lower()})" + if not isinstance(num_rows, int) + else None, + byte_count=byte_count if isinstance(byte_count, int) else None, + byte_count_basis="tables_api_num_bytes" if isinstance(byte_count, int) else None, + ) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 0820e86f25..7a4bf15518 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -465,22 +465,34 @@ def physical_relation_generation( return generation_comment(tuple(row)).removeprefix(GENERATION_COMMENT_PREFIX) def estimate_relation_row_count( - self, *, connection: Any, database: str | None, schema: str | None, name: str + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: del database row: Any = self.execute( connection=connection, - sql="SELECT engine, total_rows FROM system.tables WHERE " + sql=( + "SELECT engine, total_rows, total_bytes FROM system.tables WHERE " + if metadata_only + else "SELECT engine, total_rows FROM system.tables WHERE " + ) + _database_filter(schema) + f" AND name = {quote_string(name)}", ).fetchone() if row is None: return RelationRowCountEstimate(row_count=None, detail="relation not found") - if row[1] is None: - return RelationRowCountEstimate( - row_count=None, detail=f"engine {row[0]} does not report row counts" - ) - return RelationRowCountEstimate(row_count=int(row[1])) + byte_count: int | None = int(row[2]) if metadata_only and row[2] is not None else None + return RelationRowCountEstimate( + row_count=int(row[1]) if row[1] is not None else None, + detail=f"engine {row[0]} does not report row counts" if row[1] is None else None, + byte_count=byte_count, + byte_count_basis="system_tables_total_bytes" if byte_count is not None else None, + ) def read_view_definition( self, *, connection: Any, database: str | None, schema: str, name: str diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index edf2f88a2f..a7eae3813d 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -2565,6 +2565,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read ``ANALYZE TABLE`` statistics; ``DESCRIBE DETAIL`` has no row count.""" @@ -2596,12 +2597,19 @@ def estimate_relation_row_count( statistics_match: re.Match[str] | None = re.search( r"(\d+)\s+rows", details.get("statistics", "") ) - if statistics_match is None: - return RelationRowCountEstimate( - row_count=None, - detail="table statistics not collected; run ANALYZE TABLE ... COMPUTE STATISTICS", - ) - return RelationRowCountEstimate(row_count=int(statistics_match.group(1))) + bytes_match: re.Match[str] | None = ( + re.search(r"(\d+)\s+bytes", details.get("statistics", "")) if metadata_only else None + ) + return RelationRowCountEstimate( + row_count=int(statistics_match.group(1)) if statistics_match is not None else None, + detail=( + "table statistics not collected; run ANALYZE TABLE ... COMPUTE STATISTICS" + if statistics_match is None + else None + ), + byte_count=int(bytes_match.group(1)) if bytes_match is not None else None, + byte_count_basis="describe_table_statistics_bytes" if bytes_match is not None else None, + ) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 3232567671..05bb55cd76 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -2356,6 +2356,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read the planner's ``reltuples`` estimate, treating unanalyzed data as unknown.""" @@ -2381,12 +2382,16 @@ def estimate_relation_row_count( row_count=None, detail="no table statistics (view or partitioned table)" ) estimated_rows: float = float(row[1]) + row_count: int | None = None if estimated_rows > 0: - return RelationRowCountEstimate(row_count=int(estimated_rows)) - if int(row[2]) == 0: - return RelationRowCountEstimate(row_count=0) + row_count = int(estimated_rows) + elif int(row[2]) == 0: + row_count = 0 return RelationRowCountEstimate( - row_count=None, detail="table statistics not collected; run ANALYZE" + row_count=row_count, + detail="table statistics not collected; run ANALYZE" if row_count is None else None, + byte_count=int(row[2]) if metadata_only else None, + byte_count_basis="pg_relation_size_main_fork" if metadata_only else None, ) def diff_schema( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 49736eb0af..c66c388a16 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3548,6 +3548,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read ``ROW_COUNT`` from the information schema; views report no count.""" @@ -3556,13 +3557,17 @@ def estimate_relation_row_count( database=database, schema=schema, name=name, - projection="row_count", + projection="row_count,bytes" if metadata_only else "row_count", ) if row is None: return RelationRowCountEstimate(row_count=None, detail="relation not found") - if row[0] is None: - return RelationRowCountEstimate(row_count=None, detail="no row count (view)") - return RelationRowCountEstimate(row_count=int(row[0])) + byte_count: int | None = int(row[1]) if metadata_only and row[1] is not None else None + return RelationRowCountEstimate( + row_count=int(row[0]) if row[0] is not None else None, + detail="no row count (view)" if row[0] is None else None, + byte_count=byte_count, + byte_count_basis="information_schema_bytes" if byte_count is not None else None, + ) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 235d9f9169..a04a722fec 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -1502,9 +1502,14 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: - """Count table rows: a local file has no metadata count, and counting reads only pages.""" + """Keep interactive counts; background metadata sampling never reads table pages.""" + if metadata_only: + return RelationRowCountEstimate( + row_count=None, detail="SQLite does not report table row counts from metadata" + ) del database table_name: str = physical_name(schema=schema, name=name) row: tuple[Any, ...] | None = connection.execute_with( diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 25d13ae0f1..7bafe69563 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -1344,6 +1344,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Sum heap or clustered-index partition row counts from the catalog.""" diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py index 09ec18f774..64b140c19c 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -219,3 +219,10 @@ class SqliteUnkeyedSampleTestCase: limit: int expected_row_count: int expected_order_ids: frozenset[object] + + +@dataclass(frozen=True) +class SqliteStatisticsTestCase: + description: str + fixture_name: str + expected_interactive_row_count: int diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py index 389f56d9ed..5afc6b4e14 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any @@ -9,6 +10,7 @@ from sqlbuild.adapter.contract.models import ( ColumnInfo, RelationInfo, + RelationRowCountEstimate, RowDiffResult, RowDiffSampleRow, ) @@ -25,6 +27,7 @@ SqliteSchemaEvolutionTestCase, SqliteSeedTestCase, SqliteStateTableTestCase, + SqliteStatisticsTestCase, SqliteStrategyTestCase, SqliteSwapTestCase, SqliteTypedTableTestCase, @@ -766,3 +769,49 @@ def test_given_view_when_renaming_then_sqlite_recreates_it( ) == test_case.expected_names ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteStatisticsTestCase( + description="background metadata avoids scans while interactive counts remain exact", + fixture_name="sqlite", + expected_interactive_row_count=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_sqlite_rows_when_sampling_metadata_then_unknown_without_scanning( + test_case: SqliteStatisticsTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + fixture: Any = json.loads( + ( + Path(__file__).resolve().parents[6] + / "benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json" + ).read_text() + )[test_case.fixture_name] + run_sql(adapter=adapter, connection=connection, statements=tuple(fixture["setup_sql"])) + statements: list[str] = [] + connection.raw_connection.set_trace_callback(statements.append) + + metadata: RelationRowCountEstimate = adapter.estimate_relation_row_count( + connection=connection, + database=None, + schema="main", + name=fixture["name"], + metadata_only=True, + ) + + assert metadata.row_count is None + assert metadata.detail == fixture["metadata_detail"] + assert metadata.byte_count is None + assert metadata.byte_count_basis is None + assert statements == [] + + interactive: RelationRowCountEstimate = adapter.estimate_relation_row_count( + connection=connection, database=None, schema="main", name=fixture["name"] + ) + + assert interactive.row_count == test_case.expected_interactive_row_count + assert interactive.detail is None diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index b54f5f8247..d86173bd94 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -145,6 +145,9 @@ class RelationRowCountEstimateTestCase: expected_sql_fragment: str database: str | None = "analytics" schema: str | None = "dev" + metadata_only: bool = False + expected_byte_count: int | None = None + expected_byte_count_basis: str | None = None @dataclass(frozen=True) @@ -154,6 +157,9 @@ class BigQueryRowCountEstimateTestCase: num_rows: int | None expected_row_count: int | None expected_table_id: str = "proj.dev.orders" + metadata_only: bool = False + num_bytes: int | None = None + expected_byte_count: int | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapters/helpers.py b/tests/unit/src/sqlbuild/adapters/helpers.py index e5d33e4c46..bc838d5b69 100644 --- a/tests/unit/src/sqlbuild/adapters/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/helpers.py @@ -118,6 +118,7 @@ class FakeBigQueryTable: table_type: str num_rows: int | None + num_bytes: int | None = None class FakeBigQueryClient: diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py b/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py index a286283dff..dd28c03fbe 100644 --- a/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py +++ b/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py @@ -9,6 +9,7 @@ from sqlbuild.adapter.contract.models import RelationRowCountEstimate from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter @@ -159,6 +160,80 @@ expected_detail="relation not found", expected_sql_fragment="schemas.name = N'dev'", ), + RelationRowCountEstimateTestCase( + description="postgres bytes remain available without analyzed rows", + adapter=PostgresAdapter(), + metadata_rows=(("r", -1.0, 8192),), + expected_row_count=None, + expected_detail="table statistics not collected; run ANALYZE", + expected_sql_fragment="pg_relation_size", + metadata_only=True, + expected_byte_count=8192, + expected_byte_count_basis="pg_relation_size_main_fork", + ), + RelationRowCountEstimateTestCase( + description="postgres empty storage is a known zero", + adapter=PostgresAdapter(), + metadata_rows=(("r", -1.0, 0),), + expected_row_count=0, + expected_detail=None, + expected_sql_fragment="pg_relation_size", + metadata_only=True, + expected_byte_count=0, + expected_byte_count_basis="pg_relation_size_main_fork", + ), + RelationRowCountEstimateTestCase( + description="snowflake bytes and rows are distinct", + adapter=SnowflakeAdapter(), + metadata_rows=((17, 4096),), + expected_row_count=17, + expected_detail=None, + expected_sql_fragment="SELECT row_count,bytes FROM", + metadata_only=True, + expected_byte_count=4096, + expected_byte_count_basis="information_schema_bytes", + ), + RelationRowCountEstimateTestCase( + description="snowflake missing view statistics remain unavailable", + adapter=SnowflakeAdapter(), + metadata_rows=((None, None),), + expected_row_count=None, + expected_detail="no row count (view)", + expected_sql_fragment="SELECT row_count,bytes FROM", + metadata_only=True, + ), + RelationRowCountEstimateTestCase( + description="databricks bytes do not imply an analyzed row count", + adapter=DatabricksAdapter(), + metadata_rows=_DATABRICKS_DETAIL_HEADER + + (("Type", "MANAGED", ""), ("Statistics", "2048 bytes", "")), + expected_row_count=None, + expected_detail="table statistics not collected; run ANALYZE TABLE ... COMPUTE STATISTICS", + expected_sql_fragment="DESCRIBE TABLE EXTENDED", + metadata_only=True, + expected_byte_count=2048, + expected_byte_count_basis="describe_table_statistics_bytes", + ), + RelationRowCountEstimateTestCase( + description="clickhouse bytes do not imply a known row count", + adapter=ClickHouseAdapter(), + metadata_rows=(("MergeTree", None, 16384),), + expected_row_count=None, + expected_detail="engine MergeTree does not report row counts", + expected_sql_fragment="SELECT engine, total_rows, total_bytes FROM system.tables", + metadata_only=True, + expected_byte_count=16384, + expected_byte_count_basis="system_tables_total_bytes", + ), + RelationRowCountEstimateTestCase( + description="clickhouse missing bytes do not imply empty storage", + adapter=ClickHouseAdapter(), + metadata_rows=(("Memory", 3, None),), + expected_row_count=3, + expected_detail=None, + expected_sql_fragment="SELECT engine, total_rows, total_bytes FROM system.tables", + metadata_only=True, + ), ], ids=lambda case: case.description, ) @@ -174,10 +249,13 @@ def test_given_catalog_metadata_when_estimating_row_count_then_reads_only_metada database=test_case.database, schema=test_case.schema, name="orders", + metadata_only=test_case.metadata_only, ) assert estimate.row_count == test_case.expected_row_count assert estimate.detail == test_case.expected_detail + assert estimate.byte_count == test_case.expected_byte_count + assert estimate.byte_count_basis == test_case.expected_byte_count_basis assert len(connection.executed) == 1 assert test_case.expected_sql_fragment in connection.executed[0] @@ -203,6 +281,23 @@ def test_given_catalog_metadata_when_estimating_row_count_then_reads_only_metada num_rows=0, expected_row_count=None, ), + BigQueryRowCountEstimateTestCase( + description="native bytes remain available without row statistics", + table_type="TABLE", + num_rows=None, + num_bytes=32768, + metadata_only=True, + expected_row_count=None, + expected_byte_count=32768, + ), + BigQueryRowCountEstimateTestCase( + description="external bytes are not native-table storage", + table_type="EXTERNAL", + num_rows=12, + num_bytes=65536, + metadata_only=True, + expected_row_count=None, + ), ], ids=lambda case: case.description, ) @@ -210,15 +305,24 @@ def test_given_bigquery_table_metadata_when_estimating_row_count_then_uses_table test_case: BigQueryRowCountEstimateTestCase, ) -> None: client: FakeBigQueryClient = FakeBigQueryClient( - FakeBigQueryTable(table_type=test_case.table_type, num_rows=test_case.num_rows) + FakeBigQueryTable( + table_type=test_case.table_type, + num_rows=test_case.num_rows, + num_bytes=test_case.num_bytes, + ) ) connection: Any = cast(Any, FakeBigQueryConnection(client=client)) estimate: RelationRowCountEstimate = BigQueryAdapter().estimate_relation_row_count( - connection=connection, database="proj", schema="dev", name="orders" + connection=connection, + database="proj", + schema="dev", + name="orders", + metadata_only=test_case.metadata_only, ) assert estimate.row_count == test_case.expected_row_count + assert estimate.byte_count == test_case.expected_byte_count assert client.requested == [test_case.expected_table_id] From 2af07abe9d3c18fdcfa6aa8468859cf2ac2981e5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:05:14 +0000 Subject: [PATCH 105/138] feat: collect hourly warehouse metadata statistics --- .../orchestration/warehouse_statistics.json | 24 ++ clients/typescript/src/schema.d.ts | 260 +++++++++++++++++- .../src/native_commands/_helpers/execution.rs | 64 ++++- .../main/_location_metadata.rs | 29 +- .../src/native_commands/models.rs | 22 +- .../authentication/password_sessions.rs | 4 + .../src/api/_helpers/resources/changes.rs | 20 +- .../_helpers/resources/person_operations.rs | 29 +- .../src/api/_helpers/resources/router.rs | 8 + .../_helpers/resources/warehouse_metadata.rs | 40 ++- .../src/api/main/serve_api.rs | 4 +- .../sqlbuild-orchestration/src/api/models.rs | 3 + .../src/platform/_helpers/work_client.rs | 21 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/errors.rs | 16 ++ .../src/store/main/migrate_store.rs | 5 +- .../src/store/warehouse_statistics.sql | 16 ++ .../src/warehouse_metadata/_helpers/mod.rs | 1 + .../warehouse_metadata/_helpers/statistics.rs | 166 +++++++++++ .../src/warehouse_metadata/constants.rs | 1 + .../main/ingest_statistics.rs | 74 +++++ .../src/warehouse_metadata/main/mod.rs | 4 + .../main/read_person_statistics.rs | 35 +++ .../main/read_statistics_due.rs | 58 ++++ .../warehouse_metadata/main/read_watermark.rs | 7 +- .../main/run_statistics_retention.rs | 26 ++ .../src/warehouse_metadata/models.rs | 73 ++++- .../api/statistics/eligibility/helpers.rs | 105 +++++++ .../tests/api/statistics/helpers.rs | 232 ++++++++++++++++ .../tests/api/statistics/retention/helpers.rs | 88 ++++++ .../tests/api/test_catalog_reads.rs | 27 ++ .../tests/api/test_contracts.rs | 8 + .../tests/api/test_statistics_recovery.rs | 82 ++++++ .../tests/store/helpers.rs | 3 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/warehouse_metadata.rs | 9 +- .../native_host/warehouse_statistics.rs | 61 ++++ .../publication/_test_types.py | 7 + .../publication/catalog/helpers.py | 20 ++ .../publication/person/helpers.py | 27 ++ .../publication/test_statistics.py | 101 +++++++ .../publication/test_warehouse_catalog.py | 22 +- 42 files changed, 1758 insertions(+), 46 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_statistics.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json index 0926e6a039..640a05f0b8 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json @@ -1,4 +1,28 @@ { + "retained_hours": 205, + "expired_samples": 205, + "drop_during_sampling": { + "name": "orders_🧾", + "create_sql": "CREATE TABLE \"orders_🧾\" (order_id INTEGER)", + "drop_sql": "DROP TABLE \"orders_🧾\"", + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "failure": "relation disappeared before statistics sampling" + }, + "cleanup_interruption": { + "block_sql": "CREATE FUNCTION pause_statistics_cleanup() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(30); RETURN NULL; END $$; CREATE TRIGGER pause_statistics_cleanup BEFORE DELETE ON sqb_warehouse_statistics FOR EACH STATEMENT EXECUTE FUNCTION pause_statistics_cleanup();", + "wait_sql": "DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '5 seconds'; BEGIN WHILE NOT EXISTS (SELECT 1 FROM pg_locks l JOIN pg_stat_activity a ON a.pid=l.pid WHERE l.relation='sqb_warehouse_statistics'::regclass AND a.wait_event='PgSleep' AND a.query LIKE 'DELETE FROM sqb_warehouse_statistics%') LOOP IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'cleanup did not start'; END IF; PERFORM pg_stat_clear_snapshot(); PERFORM pg_sleep(0.02); END LOOP; END $$;", + "restore_sql": "DROP TRIGGER pause_statistics_cleanup ON sqb_warehouse_statistics; DROP FUNCTION pause_statistics_cleanup();" + }, + "record": { + "state_database": null, + "state_schema": "main", + "name": "orders_🧾_000", + "observed_at": "2026-01-01T00:00:00Z", + "row_count": 7, + "detail": "synthetic catalog estimate", + "byte_count": 4096, + "byte_count_basis": "synthetic_catalog_bytes" + }, "sqlite": { "setup_sql": [ "CREATE TABLE orders (order_id INTEGER)", diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index fe88e0b2aa..406e26850e 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -227,6 +227,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/statistics": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonStatistics"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/runs": { parameters: { query?: never; @@ -451,6 +467,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/statistics": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_statistics"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["statistics_due"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/watermark": { parameters: { query?: never; @@ -1496,6 +1544,40 @@ export interface components { /** Format: uuid */ session_id: string; }; + StatisticsBatch: { + /** Format: int64 */ + generation: number; + records: components["schemas"]["WarehouseStatisticsRecord"][]; + /** Format: uuid */ + session_id: string; + }; + /** @description Eligibility comes from each relation's committed hour, never a namespace watermark. */ + StatisticsDueRead: { + /** Format: uuid */ + after?: string | null; + /** Format: int64 */ + generation: number; + limit: number; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: uuid */ + session_id: string; + }; + StatisticsObservation: { + /** Format: uuid */ + event_id: string; + record: components["schemas"]["WarehouseStatisticsRecord"]; + }; + StatisticsPage: { + next_after?: null | components["schemas"]["WarehouseReconciliationCursor"]; + records: components["schemas"]["StatisticsObservation"][]; + }; + /** @description One bounded catalog page; sampled and unsampled relations both advance its cursor. */ + StatisticsRead: { + location: components["schemas"]["WarehouseStateLocation"]; + names: string[]; + /** Format: uuid */ + next_after?: string | null; + }; /** * @description The tables Recall writes. * @enum {string} @@ -1684,7 +1766,7 @@ export interface components { * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -1828,6 +1910,20 @@ export interface components { database?: string | null; schema: string; }; + /** @description A metadata estimate at collection time, never evidence of source-data currency. */ + WarehouseStatisticsRecord: { + /** Format: int64 */ + byte_count?: number | null; + byte_count_basis?: string | null; + detail?: string | null; + name: string; + /** Format: date-time */ + observed_at: string; + /** Format: int64 */ + row_count?: number | null; + state_database?: string | null; + state_schema: string; + }; /** * @description Closed failure stages keep warehouse errors and credentials out of health observations. * @enum {string} @@ -2398,6 +2494,58 @@ export interface operations { }; }; }; + readPersonStatistics: { + parameters: { + query: { + /** @description Exact database; omitted matches SQL NULL */ + database?: string; + schema: string; + name: string; + after_observed_at?: string; + after_event_id?: string; + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatisticsPage"]; + }; + }; + /** @description Invalid relation or cursor */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; requestJobRun: { parameters: { query?: never; @@ -3068,6 +3216,116 @@ export interface operations { }; }; }; + report_statistics: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["StatisticsBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + statistics_due: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["StatisticsDueRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatisticsRead"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; watermark: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 5f5fe919a8..01e48afdb5 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -31,8 +31,8 @@ use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryKind, WarehouseHistoryPage, WarehouseReconciliationCursor, - WarehouseStateLocation, WarehouseSyncResponse, + StatisticsDueRead, StatisticsRead, WarehouseHistoryKind, WarehouseHistoryPage, + WarehouseReconciliationCursor, WarehouseStateLocation, WarehouseSyncResponse, }; use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; use sqlbuild_orchestration::webhooks::main::{ @@ -333,6 +333,7 @@ pub(crate) fn metadata_sync( source_after: None, observed_since: None, watermark_loaded: false, + statistics: None, reconcile: true, reconcile_at: Instant::now(), reconcile_seconds, @@ -350,12 +351,42 @@ pub(crate) fn history_request(state: &LocationMetadataSync) -> Result, C )])); } let kind = state.kind().as_str(); - let request = json!({(kind): { - "location":state.location(),"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit - }}); + let request = if state.kind() == WarehouseHistoryKind::Statistics { + json!({(kind): state.statistics.as_ref().ok_or_else(|| ControlError::Invalid("statistics eligibility page missing".into()))?}) + } else { + json!({(kind): { + "location":state.location(),"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit + }}) + }; serde_json::to_vec(&request).map_err(|error| ControlError::Invalid(error.to_string())) } +pub(crate) async fn statistics_page( + (client, target): (&LocationWorkClient, &str), + (state, poll): (&LocationMetadataSync, &LocationWorkPoll), +) -> Result { + let after = state + .source_after + .as_ref() + .and_then(|values| values.first()) + .and_then(|value| value.as_deref()) + .map(Uuid::parse_str) + .transpose() + .map_err(|_| ClientError::Configuration("invalid statistics eligibility cursor"))?; + client + .statistics_due( + target, + &StatisticsDueRead { + session_id: poll.session_id, + generation: state.generation, + location: state.location().clone(), + after, + limit: state.limit, + }, + ) + .await +} + /// Reserve the largest mirror cursor before freezing: receipts prohibit later content splitting. pub(crate) fn freeze_history_page( mut state: LocationMetadataSync, @@ -363,11 +394,13 @@ pub(crate) fn freeze_history_page( poll: &LocationWorkPoll, ) -> Result { let location = state.location(); - let mut body = json!({"session_id":poll.session_id,"generation":state.generation,"records":page.records, - "reconciliation":{"scan_id":state.scan_id,"page":state.page,"state_database":location.database, + let mut body = + json!({"session_id":poll.session_id,"generation":state.generation,"records":page.records}); + if state.reconciling() { + body["reconciliation"] = json!({"scan_id":state.scan_id,"page":state.page,"state_database":location.database, "state_schema":location.schema,"day":null,"complete":page.next_after.is_none(), - "after":WarehouseReconciliationCursor { observed_at: DateTime::::MAX_UTC, event_id: Uuid::nil() }}}); - if !state.reconciling() { + "after":WarehouseReconciliationCursor { observed_at: DateTime::::MAX_UTC, event_id: Uuid::nil() }}); + } else if state.kind() != WarehouseHistoryKind::Statistics { body["reconciliation"] = serde_json::Value::Null; } if serde_json::to_vec(&body) @@ -375,6 +408,7 @@ pub(crate) fn freeze_history_page( .len() > BODY_BYTES { + state.statistics = None; if state.limit > 1 { state.limit = (state.limit / 2).max(1); } else { @@ -402,6 +436,15 @@ pub(crate) async fn deliver_metadata_page( kind: WarehouseHistoryKind, ) -> Result { match kind { + WarehouseHistoryKind::Statistics => { + client + .sync_statistics( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid statistics batch"))?, + ) + .await + } WarehouseHistoryKind::Catalog => { client .sync_catalog( @@ -490,6 +533,9 @@ pub(crate) fn acknowledge_metadata_page( state.pending = Some(pending); } else { state.source_after = pending.next_source_after; + if state.kind() == WarehouseHistoryKind::Statistics { + state.statistics = None; + } if state.source_after.is_some() { state.page = state .page diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index e93c96caf2..973c5376b3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -17,7 +17,7 @@ use tokio::time::Instant; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ acknowledge_metadata_page, deliver_metadata_page, freeze_history_page, history_request, - location_result, metadata_health, + location_result, metadata_health, statistics_page, }; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; @@ -42,11 +42,25 @@ pub(super) async fn sync_location_metadata( let started = Instant::now(); tokio::pin!(stopped); if state.pending.is_none() { + if state.kind() == WarehouseHistoryKind::Statistics && state.statistics.is_none() { + state.statistics = Some(tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = statistics_page((client, &options.target), (&state, poll)) => match result { + Ok(page) => page, + Err(error) => return retry_metadata(state, error), + }, + }); + } if !state.watermark_loaded { if state.index == 0 { state.reconcile = Instant::now() >= state.reconcile_at; } - if !state.reconcile && state.kind() != WarehouseHistoryKind::Catalog { + if !state.reconcile + && !matches!( + state.kind(), + WarehouseHistoryKind::Catalog | WarehouseHistoryKind::Statistics + ) + { let request = WarehouseWatermarkRead { session_id: poll.session_id, generation: state.generation, @@ -82,8 +96,7 @@ pub(super) async fn sync_location_metadata( result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Read))) => {}, } - state.limit = (state.limit / 2).max(1); - state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + state.retry_read(); drop(write_output(&[( Stream::Stderr, "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", @@ -145,6 +158,14 @@ async fn read_metadata_page( state: &LocationMetadataSync, stopped: impl Future>, ) -> Result>, ControlError> { + if let Some(statistics) = &state.statistics + && statistics.names.is_empty() + { + return Ok(Some(Some(WarehouseHistoryPage { + records: Vec::new(), + next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]), + }))); + } let (mut child, reply) = match location_result::>( (options, script), (history_request(state)?, METADATA_REPLY_BYTES), diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index b609c33f17..597de71a37 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -9,7 +9,7 @@ use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryKind, WarehouseStateLocation, + StatisticsRead, WarehouseHistoryKind, WarehouseStateLocation, }; use uuid::Uuid; @@ -195,6 +195,7 @@ pub(crate) struct LocationMetadataSync { pub(crate) source_after: Option>>, pub(crate) observed_since: Option>, pub(crate) watermark_loaded: bool, + pub(crate) statistics: Option, pub(crate) reconcile: bool, pub(crate) reconcile_at: tokio::time::Instant, pub(crate) reconcile_seconds: u32, @@ -204,8 +205,25 @@ pub(crate) struct LocationMetadataSync { } impl LocationMetadataSync { + /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. + pub(crate) fn retry_read(&mut self) { + self.limit = (self.limit / 2).max(1); + self.statistics = None; + if self.kind() == WarehouseHistoryKind::Statistics { + self.index -= 1; + self.source_after = None; + self.scan_id = Uuid::now_v7(); + self.page = 0; + self.observed_since = None; + self.watermark_loaded = false; + } + self.next_at = tokio::time::Instant::now() + + std::time::Duration::from_secs(super::constants::METADATA_POLL_SECONDS); + } + pub(crate) fn reconciling(&self) -> bool { - self.reconcile || self.kind() == WarehouseHistoryKind::Catalog + self.kind() != WarehouseHistoryKind::Statistics + && (self.reconcile || self.kind() == WarehouseHistoryKind::Catalog) } pub(crate) fn kind(&self) -> WarehouseHistoryKind { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 9bc2ab2128..c758b4443d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -66,6 +66,10 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/control/locations/{location_id}/targets/{target}/catalog", get(crate::api::_helpers::resources::person_operations::catalog), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/statistics", + get(crate::api::_helpers::resources::person_operations::statistics), + ) .route( "/api/control/locations/{location_id}/targets/{target}/fingerprint", get(crate::api::_helpers::resources::person_operations::fingerprint), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs index 3d5ffb7424..70e341de64 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs @@ -142,7 +142,9 @@ pub(crate) async fn relay( let _ = sender.send(ChangeMessage::Resync); Some(listener) } - Err(ControlError::Database(error)) if transient_listener_error(&error) => continue, + Err(ControlError::Database(error)) if ControlError::transient_database(&error) => { + continue; + } Err(error) => return Err(error), }; } @@ -153,7 +155,7 @@ pub(crate) async fn relay( Ok(Some(notification)) => serde_json::from_str(notification.payload()) .map_err(|error| ControlError::Invalid(error.to_string()))?, Ok(None) => ChangeMessage::Resync, - Err(error) if transient_listener_error(&error) => { + Err(error) if ControlError::transient_database(&error) => { listener = None; continue; } @@ -162,17 +164,3 @@ pub(crate) async fn relay( let _ = sender.send(message); } } - -fn transient_listener_error(error: &sqlx_core::Error) -> bool { - match error { - sqlx_core::Error::Io(_) - | sqlx_core::Error::PoolTimedOut - | sqlx_core::Error::WorkerCrashed => true, - sqlx_core::Error::Database(error) => error.code().is_some_and(|code| { - code.starts_with("08") - || code.starts_with("53") - || matches!(code.as_ref(), "57P01" | "57P02" | "57P03") - }), - _ => false, - } -} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 32708a27c7..aca4c6b73c 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -20,11 +20,38 @@ use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; +use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{ CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, - NodeStatusQuery, + NodeStatusQuery, StatisticsPage, StatisticsQuery, }; +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("database" = Option, Query, description = "Exact database; omitted matches SQL NULL"), + ("schema" = String, Query), ("name" = String, Query), + ("after_observed_at" = Option>, Query), + ("after_event_id" = Option, Query), ("limit" = Option, Query)), + responses((status = 200, body = StatisticsPage), (status = 400, description = "Invalid relation or cursor"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn statistics( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + headers: HeaderMap, +) -> Result, ApiError> { + Ok(Json( + read_person_statistics( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} + #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 23517df972..1143bfe060 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -63,6 +63,14 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_statistics_due(&store, (location_id, &target), &request).await?, + )) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/statistics", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = StatisticsBatch, + responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Reader generation fenced")), security(("bearer" = [])))] +pub(crate) async fn report_statistics( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_statistics(&store, (location_id, &target), &batch).await?, + )) +} + #[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/catalog", params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = CatalogBatch, responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 4d0e9ec477..d9d0d2029d 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -11,6 +11,7 @@ use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::run_statistics_retention::run_statistics_retention; pub async fn serve_api( store: ControlStore, @@ -51,7 +52,8 @@ pub async fn serve_api( ready(); tokio::select! { result = async { server.await } => { result?; Ok(()) }, - result = changes::relay(store, listener_changes, sender) => result, + result = changes::relay(store.clone(), listener_changes, sender) => result, + result = run_statistics_retention(&store) => result, } } diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c5d71e4f3e..1f5162f8f7 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -21,6 +21,7 @@ use crate::api::_helpers::resources::{ person_operations::change_trigger, person_operations::recent_runs, person_operations::catalog, + person_operations::statistics, person_operations::fingerprint, person_operations::node_status, password_sessions::login, @@ -59,6 +60,8 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report_old_name_views, warehouse_metadata::report_microbatches, warehouse_metadata::report_catalog, + warehouse_metadata::report_statistics, + warehouse_metadata::statistics_due, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index f83364e585..f530db781e 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -11,11 +11,28 @@ use crate::platform::models::{ }; use crate::warehouse_metadata::models::{ AuditResultsBatch, CatalogBatch, FingerprintBatch, JanitorEventsBatch, MicrobatchesBatch, - MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, WarehouseSyncResponse, - WarehouseWatermarkRead, + MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, StatisticsBatch, StatisticsDueRead, + StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { + pub async fn statistics_due( + &self, + target: &str, + request: &StatisticsDueRead, + ) -> Result { + self.sync_metadata((target, "statistics-due"), request) + .await + } + + pub async fn sync_statistics( + &self, + target: &str, + batch: &StatisticsBatch, + ) -> Result { + self.sync_metadata((target, "statistics"), batch).await + } + pub async fn sync_catalog( &self, target: &str, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index c9a42eb73a..7715b48e4c 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -36,6 +36,7 @@ pub(crate) const SOURCE_STALENESS: &str = include_str!("../source_staleness.sql" pub(crate) const SECURITY_AUDIT_SINK: &str = include_str!("../security_audit_sink.sql"); pub(crate) const WAREHOUSE_CATALOG: &str = include_str!("../warehouse_catalog.sql"); pub(crate) const FINGERPRINT_LOOKUP: &str = include_str!("../fingerprint_lookup.sql"); +pub(crate) const WAREHOUSE_STATISTICS: &str = include_str!("../warehouse_statistics.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/errors.rs b/crates/sqlbuild-orchestration/src/store/errors.rs index 41af8db6fc..8d552fd243 100644 --- a/crates/sqlbuild-orchestration/src/store/errors.rs +++ b/crates/sqlbuild-orchestration/src/store/errors.rs @@ -25,3 +25,19 @@ pub enum ControlError { #[error("scheduler lease expired or was fenced by another holder")] Fenced, } + +impl ControlError { + pub(crate) fn transient_database(error: &sqlx_core::Error) -> bool { + match error { + sqlx_core::Error::Io(_) + | sqlx_core::Error::PoolTimedOut + | sqlx_core::Error::WorkerCrashed => true, + sqlx_core::Error::Database(error) => error.code().is_some_and(|code| { + code.starts_with("08") + || code.starts_with("53") + || matches!(code.as_ref(), "57P01" | "57P02" | "57P03") + }), + _ => false, + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 51c4bb1067..54e98409db 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -10,8 +10,8 @@ use crate::store::_helpers::migration::{ PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, SECURITY_AUDIT_SINK, SENSORS, SOURCE_SENSORS, SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, WAREHOUSE_CATALOG, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, - WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, render, - verify_migrations, + WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, WAREHOUSE_STATISTICS, + render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -64,6 +64,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("security_audit_sink", SECURITY_AUDIT_SINK), ("warehouse_catalog", WAREHOUSE_CATALOG), ("fingerprint_lookup", FINGERPRINT_LOOKUP), + ("warehouse_statistics", WAREHOUSE_STATISTICS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_statistics.sql b/crates/sqlbuild-orchestration/src/store/warehouse_statistics.sql new file mode 100644 index 0000000000..f5aeed1627 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_statistics.sql @@ -0,0 +1,16 @@ +CREATE TABLE sqb_warehouse_statistics ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + relation_name TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_statistics_relation_time ON sqb_warehouse_statistics + (location_id,target,state_schema,state_database,relation_name,observed_at,event_id); +CREATE INDEX sqb_statistics_expiration ON sqb_warehouse_statistics (observed_at,event_id); +CREATE INDEX sqb_catalog_statistics_scan ON sqb_warehouse_catalog + (location_id,target,state_schema,state_database,event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs index b5049128dc..b331f7fff9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -7,3 +7,4 @@ pub(crate) mod ingestion; pub(crate) mod node_status; pub(crate) mod person_reads; pub(crate) mod reconciliation; +pub(crate) mod statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs new file mode 100644 index 0000000000..0e4a885fe4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs @@ -0,0 +1,166 @@ +//! Exact relation/time selection never joins definitions or infers data currency. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Duration, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::store::types::Row; +use crate::warehouse_metadata::_helpers::ingestion::scoped_event_id; +use crate::warehouse_metadata::constants::{MAX_HISTORY_RECORDS, STATISTICS_RETENTION_DAYS}; +use crate::warehouse_metadata::models::{ + StatisticsObservation, StatisticsPage, StatisticsQuery, WarehouseCatalogRecord, + WarehouseReconciliationCursor, +}; + +pub(crate) fn statistics_event_id( + scope: (Uuid, &str), + (database, schema, name): (Option<&str>, &str, &str), + observed_at: DateTime, +) -> Result { + scoped_event_id( + scope, + (database, schema, "statistics"), + &json!((name, observed_at.timestamp().div_euclid(3600))).to_string(), + ) +} + +/// Expiration is bounded global cleanup, never a lifecycle status or warehouse write. +pub(crate) async fn prune_statistics(store: &ControlStore) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let cutoff = transaction.now - Duration::days(STATISTICS_RETENTION_DAYS); + let (transaction, rows) = query(transaction, + "DELETE FROM sqb_warehouse_statistics WHERE event_id IN (SELECT event_id FROM sqb_warehouse_statistics \ + WHERE observed_at<$1 ORDER BY observed_at,event_id LIMIT $2) RETURNING event_id", + vec![Parameter::Time(cutoff), Parameter::Integer(MAX_HISTORY_RECORDS as i64)]).await?; + commit(transaction).await?; + Ok(rows.len()) +} + +/// A partial namespace page never suppresses another relation's missing current-hour sample. +pub(crate) async fn eligible_names( + mut transaction: ControlTransaction, + scope: (Uuid, &str), + rows: Vec, +) -> Result<(ControlTransaction, Vec), ControlError> { + let mut candidates: Vec<(Uuid, String)> = Vec::new(); + for row in rows { + let record: WarehouseCatalogRecord = serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let id = statistics_event_id( + scope, + ( + record.state_database.as_deref(), + &record.state_schema, + &record.name, + ), + transaction.now, + )?; + candidates.push((id, record.name)); + } + let mut committed: BTreeSet = BTreeSet::new(); + if !candidates.is_empty() { + let placeholders = (1..=candidates.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let found; + (transaction, found) = query( + transaction, + &format!( + "SELECT event_id FROM sqb_warehouse_statistics WHERE event_id IN ({placeholders})" + ), + candidates + .iter() + .map(|(id, _)| Parameter::Id(*id)) + .collect(), + ) + .await?; + for row in found { + committed.insert( + serde_json::from_value::(row["event_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + } + } + Ok(( + transaction, + candidates + .into_iter() + .filter(|(id, _)| !committed.contains(id)) + .map(|(_, name)| name) + .collect(), + )) +} + +pub(crate) async fn read_statistics_page( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + options: &StatisticsQuery, +) -> Result<(ControlTransaction, StatisticsPage), ControlError> { + if options.schema.trim().is_empty() + || options.name.trim().is_empty() + || options.after_observed_at.is_some() != options.after_event_id.is_some() + { + return Err(ControlError::Invalid( + "statistics reads need an exact relation and both cursor fields or neither".into(), + )); + } + let limit = options + .limit + .unwrap_or(MAX_HISTORY_RECORDS as u32) + .clamp(1, MAX_HISTORY_RECORDS as u32) as usize; + let cutoff = transaction.now - Duration::days(STATISTICS_RETENTION_DAYS); + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.schema.clone()), + Parameter::Text(options.name.clone()), + Parameter::Time(cutoff), + ]; + let database = if let Some(database) = &options.database { + parameters.push(Parameter::Text(database.clone())); + "state_database=$6" + } else { + "state_database IS NULL" + }; + let mut after = String::new(); + if let (Some(time), Some(id)) = (options.after_observed_at, options.after_event_id) { + let offset = parameters.len(); + parameters.extend([Parameter::Time(time), Parameter::Id(id)]); + after = format!( + " AND (observed_at,event_id)>(${},${})", + offset + 1, + offset + 2 + ); + } + parameters.push(Parameter::Integer(limit as i64)); + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,record_json AS record FROM sqb_warehouse_statistics \ + WHERE location_id=$1 AND target=$2 AND state_schema=$3 AND relation_name=$4 AND observed_at>=$5 \ + AND {database}{after} ORDER BY observed_at,event_id LIMIT ${}", parameters.len()), parameters).await?; + let more = rows.len() == limit; + let records: Vec = rows + .into_iter() + .map(|row| serde_json::from_value(json!(row))) + .collect::>() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let next_after = + more.then(|| records.last()) + .flatten() + .map(|observation| WarehouseReconciliationCursor { + observed_at: observation.record.observed_at, + event_id: observation.event_id, + }); + Ok(( + transaction, + StatisticsPage { + records, + next_after, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 7770ce5e30..c8445bfec6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -2,3 +2,4 @@ pub const MAX_HISTORY_RECORDS: usize = 100; pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; +pub(crate) const STATISTICS_RETENTION_DAYS: i64 = 90; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs new file mode 100644 index 0000000000..b99650d98f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs @@ -0,0 +1,74 @@ +//! First committed metadata sample per relation and UTC hour wins; samples are not change facts. + +use std::collections::BTreeSet; + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, validate_fields, +}; +use crate::warehouse_metadata::_helpers::statistics::statistics_event_id; +use crate::warehouse_metadata::models::{StatisticsBatch, WarehouseSyncResponse}; + +pub async fn ingest_statistics( + store: &ControlStore, + scope: (Uuid, &str), + batch: &StatisticsBatch, +) -> Result { + let mut names: BTreeSet<_> = BTreeSet::new(); + validate_fields( + ("statistics", batch.generation, batch.records.len()), + batch.records.iter().any(|record| { + record.state_schema.trim().is_empty() + || record.name.trim().is_empty() + || record.byte_count.is_some() != record.byte_count_basis.is_some() + || record + .byte_count_basis + .as_ref() + .is_some_and(|basis| basis.trim().is_empty()) + || !names.insert(( + ( + record.state_database.as_deref(), + &record.state_schema, + &record.name, + ), + record.observed_at.timestamp().div_euclid(3600), + )) + }), + false, + )?; + let mut transaction = + begin_ingestion(store, scope, (batch.session_id, batch.generation)).await?; + let mut applied = false; + for record in &batch.records { + let event_id = statistics_event_id( + scope, + ( + record.state_database.as_deref(), + &record.state_schema, + &record.name, + ), + record.observed_at, + )?; + let now = transaction.now; + let rows; + (transaction, rows) = query(transaction, + "INSERT INTO sqb_warehouse_statistics (event_id,location_id,target,state_database,state_schema,relation_name,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id),Parameter::Id(scope.0),Parameter::Text(scope.1.into()), + Parameter::OptionalText(record.state_database.clone()),Parameter::Text(record.state_schema.clone()), + Parameter::Text(record.name.clone()),Parameter::Time(record.observed_at),Parameter::Json(json!(record)),Parameter::Time(now)]).await?; + applied |= !rows.is_empty(); + } + commit_ingestion( + transaction, + ("sqb_warehouse_statistics", scope.0, scope.1), + (None, &[]), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 3060cb58f4..c20457c46b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -7,7 +7,11 @@ pub mod ingest_microbatches; pub mod ingest_migrations; pub mod ingest_old_name_views; pub mod ingest_source_freshness; +pub mod ingest_statistics; pub mod read_person_catalog; pub mod read_person_fingerprint; pub mod read_person_node_status; +pub mod read_person_statistics; +pub mod read_statistics_due; pub mod read_watermark; +pub mod run_statistics_retention; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs new file mode 100644 index 0000000000..55e1fa3464 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs @@ -0,0 +1,35 @@ +//! Live View authority and bounded historical estimates share one writer transaction. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::_helpers::statistics::read_statistics_page; +use crate::warehouse_metadata::models::{StatisticsPage, StatisticsQuery}; + +pub async fn read_person_statistics( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + (location_id, target, options): (Uuid, &str, &StatisticsQuery), + default_role: DefaultRole, +) -> Result { + let transaction = begin_view( + store, + (context, credentials), + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + default_role, + ) + .await?; + let (transaction, page) = + read_statistics_page(transaction, (location_id, target), options).await?; + commit(transaction).await?; + Ok(page) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs new file mode 100644 index 0000000000..b8feaeb60d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs @@ -0,0 +1,58 @@ +//! Bounded catalog traversal excludes only relations with a committed sample this UTC hour. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::_helpers::statistics::eligible_names; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{StatisticsDueRead, StatisticsRead}; + +pub async fn read_statistics_due( + store: &ControlStore, + scope: (Uuid, &str), + request: &StatisticsDueRead, +) -> Result { + if request.location.schema.trim().is_empty() + || !(1..=MAX_HISTORY_RECORDS).contains(&request.limit) + { + return Err(ControlError::Invalid( + "statistics eligibility needs a namespace and a bounded page".into(), + )); + } + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let mut parameters = vec![ + Parameter::Id(scope.0), + Parameter::Text(scope.1.into()), + Parameter::Text(request.location.schema.clone()), + Parameter::Id(request.after.unwrap_or(Uuid::nil())), + ]; + let database = if let Some(database) = &request.location.database { + parameters.push(Parameter::Text(database.clone())); + "state_database=$5" + } else { + "state_database IS NULL" + }; + parameters.push(Parameter::Integer(request.limit as i64)); + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,record_json FROM sqb_warehouse_catalog WHERE location_id=$1 AND target=$2 \ + AND state_schema=$3 AND event_id>$4 AND {database} ORDER BY event_id LIMIT ${}", parameters.len()), parameters).await?; + let next_after = if rows.len() == request.limit { + rows.last() + .map(|row| serde_json::from_value(row["event_id"].clone())) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))? + } else { + None + }; + let (transaction, names) = eligible_names(transaction, scope, rows).await?; + commit(transaction).await?; + Ok(StatisticsRead { + location: request.location.clone(), + names, + next_after, + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs index 35ec55edfd..e6841fc058 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs @@ -16,9 +16,12 @@ pub async fn read_watermark( ) -> Result>, ControlError> { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; - if request.kind == WarehouseHistoryKind::Catalog { + if matches!( + request.kind, + WarehouseHistoryKind::Catalog | WarehouseHistoryKind::Statistics + ) { return Err(ControlError::Invalid( - "catalog observations require full scans, not timestamp watermarks".into(), + "catalog and statistics observations do not support timestamp watermarks".into(), )); } let table = request.kind.mirror_table(); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs new file mode 100644 index 0000000000..4d24463ce7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs @@ -0,0 +1,26 @@ +//! Independent cleanup also expires samples from removed locations and namespaces. + +use std::time::Duration; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::statistics::prune_statistics; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; + +pub async fn run_statistics_retention(store: &ControlStore) -> Result<(), ControlError> { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + interval.tick().await; + loop { + match prune_statistics(store).await { + Ok(MAX_HISTORY_RECORDS) => tokio::task::yield_now().await, + Ok(_) => break, + Err(ControlError::Database(error)) if ControlError::transient_database(&error) => { + break; + } + Err(error) => return Err(error), + } + } + } +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index bdc99b9935..e1b3f88f8c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -290,10 +290,11 @@ pub enum WarehouseHistoryKind { OldNameViews, Microbatches, Catalog, + Statistics, } impl WarehouseHistoryKind { - pub const ALL: [Self; 8] = [ + pub const ALL: [Self; 9] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -302,6 +303,7 @@ impl WarehouseHistoryKind { Self::OldNameViews, Self::Microbatches, Self::Catalog, + Self::Statistics, ]; #[must_use] @@ -315,6 +317,7 @@ impl WarehouseHistoryKind { Self::OldNameViews => "old_name_views", Self::Microbatches => "microbatches", Self::Catalog => "catalog", + Self::Statistics => "statistics", } } @@ -329,6 +332,7 @@ impl WarehouseHistoryKind { Self::OldNameViews => "sqb_warehouse_old_name_views", Self::Microbatches => "sqb_warehouse_microbatches", Self::Catalog => "sqb_warehouse_catalog", + Self::Statistics => "sqb_warehouse_statistics", } } } @@ -445,6 +449,7 @@ pub enum WarehouseHistoryRead { OldNameViews(StateEventsRead), Microbatches(StateEventsRead), Catalog(CatalogRead), + Statistics(StatisticsRead), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -458,6 +463,7 @@ pub enum WarehouseHistoryRecord { OldNameView(Box), Microbatch(Box), Catalog(WarehouseCatalogRecord), + Statistics(WarehouseStatisticsRecord), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -618,3 +624,68 @@ pub struct NodeStatus { pub build: Option, pub source_data: SourceDataStatus, } + +/// A metadata estimate at collection time, never evidence of source-data currency. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseStatisticsRecord { + pub state_database: Option, + pub state_schema: String, + pub name: String, + pub observed_at: DateTime, + pub row_count: Option, + pub detail: Option, + pub byte_count: Option, + pub byte_count_basis: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct StatisticsBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, +} + +/// Eligibility comes from each relation's committed hour, never a namespace watermark. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct StatisticsDueRead { + pub session_id: Uuid, + pub generation: i64, + pub location: WarehouseStateLocation, + pub after: Option, + pub limit: usize, +} + +/// One bounded catalog page; sampled and unsampled relations both advance its cursor. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct StatisticsRead { + pub location: WarehouseStateLocation, + pub names: Vec, + pub next_after: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StatisticsQuery { + pub database: Option, + pub schema: String, + pub name: String, + pub after_observed_at: Option>, + pub after_event_id: Option, + pub limit: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct StatisticsObservation { + pub event_id: Uuid, + pub record: WarehouseStatisticsRecord, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct StatisticsPage { + pub records: Vec, + pub next_after: Option, +} diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs new file mode 100644 index 0000000000..8401242322 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs @@ -0,0 +1,105 @@ +//! Partial namespace sampling cannot hide unsampled relations after a reader restart. + +use chrono::Utc; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::{ + ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, + read_statistics_due::read_statistics_due, +}; +use crate::warehouse_metadata::models::{ + StatisticsBatch, StatisticsDueRead, StatisticsQuery, WarehouseStateLocation, +}; + +pub(crate) async fn statistics_eligibility( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location, session_id): (Uuid, Uuid), + record: &Value, +) { + let mut request = StatisticsDueRead { + session_id, + generation: 1, + location: WarehouseStateLocation { + database: None, + schema: "main".into(), + }, + after: None, + limit: 100, + }; + let first = read_statistics_due(store, (location, "development"), &request) + .await + .expect("due first page"); + assert_eq!(first.names.len(), 100); + let mut sample = record.clone(); + sample["name"] = json!(first.names[0]); + sample["observed_at"] = json!(Utc::now()); + sample["row_count"] = Value::Null; + sample["byte_count"] = json!(0); + let batch: StatisticsBatch = + serde_json::from_value(json!({"session_id":session_id,"generation":1,"records":[sample]})) + .expect("one sampled relation"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("partial sampling") + .applied + ); + let options = StatisticsQuery { + database: None, + schema: "main".into(), + name: first.names[0].clone(), + after_observed_at: Some(batch.records[0].observed_at), + after_event_id: Some(Uuid::nil()), + limit: None, + }; + let observed = read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("unknown rows and zero bytes remain distinct"); + let current = observed.records.last().expect("current-hour sample"); + assert_eq!(current.record.row_count, None); + assert_eq!(current.record.byte_count, Some(0)); + assert_eq!( + current.record.byte_count_basis.as_deref(), + Some("synthetic_catalog_bytes") + ); + let remaining = read_statistics_due(store, (location, "development"), &request) + .await + .expect("restart from committed per-relation progress"); + assert_eq!(remaining.names, first.names[1..]); + assert_eq!(remaining.next_after, first.next_after); + request.after = remaining.next_after; + let second = read_statistics_due(store, (location, "development"), &request) + .await + .expect("next page"); + assert_eq!(second.names.len(), 100); + request.after = second.next_after; + let last = read_statistics_due(store, (location, "development"), &request) + .await + .expect("last page"); + assert_eq!(last.names.len(), 5); + assert!(last.next_after.is_none()); + request.after = None; + request.location.database = Some("orders_data".into()); + assert_eq!( + read_statistics_due(store, (location, "development"), &request) + .await + .expect("exact database") + .names, + ["orders_🧾_000"] + ); + request.generation = 2; + assert!( + read_statistics_due(store, (location, "development"), &request) + .await + .is_err() + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs new file mode 100644 index 0000000000..407156e02a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs @@ -0,0 +1,232 @@ +//! Samples are immutable hourly estimates, exact-scoped and independent of structural facts. + +use chrono::{Duration, Timelike, Utc}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_metadata::main::{ + ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, +}; +use crate::warehouse_metadata::models::{StatisticsBatch, StatisticsQuery}; + +pub(crate) async fn statistics_reads( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location, other, session_id): (Uuid, Uuid, Uuid), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json"); + let data: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("statistics fixture"); + let now = Utc::now() + .with_minute(0) + .expect("minute") + .with_second(0) + .expect("second") + .with_nanosecond(0) + .expect("nanosecond"); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("facts before"); + let (transaction, before) = query( + transaction, + "SELECT COUNT(*) AS total FROM recall_events", + vec![], + ) + .await + .expect("fact count"); + commit(transaction).await.expect("facts read"); + let records: Vec = (0..data["retained_hours"].as_i64().expect("hours")) + .map(|index| { + let mut record = data["record"].clone(); + record["observed_at"] = json!(now - Duration::hours(205 - index)); + record + }) + .collect(); + for records in records.chunks(100) { + let batch: StatisticsBatch = serde_json::from_value( + json!({"session_id":session_id,"generation":1,"records":records}), + ) + .expect("batch"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("ingest") + .applied + ); + assert!( + !ingest_statistics(store, (location, "development"), &batch) + .await + .expect("lost acknowledgement") + .applied + ); + } + let mut options = StatisticsQuery { + database: None, + schema: "main".into(), + name: "orders_🧾_000".into(), + after_observed_at: None, + after_event_id: None, + limit: Some(u32::MAX), + }; + let mut actual = Vec::new(); + for expected in [100, 100, 5] { + let page = read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("page"); + assert_eq!(page.records.len(), expected); + options.after_observed_at = page.next_after.as_ref().map(|cursor| cursor.observed_at); + options.after_event_id = page.next_after.map(|cursor| cursor.event_id); + actual.extend(page.records); + } + assert!(options.after_event_id.is_none()); + let actual_times: Vec<_> = actual + .iter() + .map(|observation| observation.record.observed_at) + .collect(); + let expected_times: Vec<_> = (1..=205) + .rev() + .map(|hours| now - Duration::hours(hours)) + .collect(); + assert_eq!(actual_times, expected_times); + assert!( + actual + .iter() + .all(|observation| observation.record.row_count == Some(7) + && observation.record.byte_count == Some(4096)) + ); + let mut replacement: StatisticsBatch = serde_json::from_value( + json!({"session_id":session_id,"generation":1,"records":[records[0]]}), + ) + .expect("replacement"); + replacement.records[0].row_count = Some(999); + replacement.records[0].observed_at += Duration::seconds(3599); + assert!( + !ingest_statistics(store, (location, "development"), &replacement) + .await + .expect("same hour first wins") + .applied + ); + for (database, schema, name) in [ + (Some("orders_data"), "main", "orders_🧾_000"), + (None, "landing", "orders_🧾_000"), + (None, "main", "Orders"), + ] { + let mut scoped = replacement.clone(); + scoped.records[0].state_database = database.map(String::from); + scoped.records[0].state_schema = schema.into(); + scoped.records[0].name = name.into(); + assert!( + ingest_statistics(store, (location, "development"), &scoped) + .await + .expect("separate exact identity") + .applied + ); + options.database = database.map(String::from); + options.schema = schema.into(); + options.name = name.into(); + assert_eq!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("exact scope") + .records + .len(), + 1 + ); + } + options.database = None; + options.schema = "main".into(); + options.name = "orders_🧾_000".into(); + options.limit = Some(1); + let first = read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("first wins"); + assert_eq!(first.records[0].record.row_count, Some(7)); + for scope in [(other, "development"), (location, "preview")] { + assert!(matches!( + read_person_statistics( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + } + options.after_event_id = Some(Uuid::now_v7()); + assert!(matches!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Invalid(_)) + )); + for (generation, reader) in [(2, session_id), (1, Uuid::now_v7())] { + let mut fenced = replacement.clone(); + fenced.generation = generation; + fenced.session_id = reader; + assert!( + ingest_statistics(store, (location, "development"), &fenced) + .await + .is_err() + ); + } + let mut invalid = replacement.clone(); + invalid.records[0].byte_count_basis = None; + assert!(matches!( + ingest_statistics(store, (location, "development"), &invalid).await, + Err(ControlError::Invalid(_)) + )); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("facts after"); + let (transaction, after) = query( + transaction, + "SELECT COUNT(*) AS total FROM recall_events", + vec![], + ) + .await + .expect("fact count"); + commit(transaction).await.expect("facts read"); + assert_eq!( + before, after, + "statistics must not append structural change facts" + ); + crate::api::tests::statistics_eligibility::statistics_eligibility( + store, + credentials, + (location, session_id), + &data["record"], + ) + .await; + crate::api::tests::statistics_retention::statistics_retention( + store, + credentials, + (location, session_id), + &data, + ) + .await; +} diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs new file mode 100644 index 0000000000..4403849958 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs @@ -0,0 +1,88 @@ +//! Bounded global expiration does not depend on catalog or namespace survival. + +use chrono::{Duration, Utc}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::statistics::prune_statistics; +use crate::warehouse_metadata::main::{ + ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, +}; +use crate::warehouse_metadata::models::{StatisticsBatch, StatisticsQuery}; + +pub(crate) async fn statistics_retention( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location, session_id): (Uuid, Uuid), + data: &Value, +) { + let records: Vec = (0..data["expired_samples"].as_i64().expect("expired samples")) + .map(|index| { + let mut record = data["record"].clone(); + record["state_schema"] = json!("removed_namespace"); + record["name"] = json!(format!("inventory_{index}")); + record["observed_at"] = json!(Utc::now() - Duration::days(91)); + record + }) + .collect(); + for records in records.chunks(100) { + let batch: StatisticsBatch = serde_json::from_value( + json!({"session_id":session_id,"generation":1,"records":records}), + ) + .expect("expired batch"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("expired sample") + .applied + ); + } + let options = StatisticsQuery { + database: None, + schema: "removed_namespace".into(), + name: "inventory_0".into(), + after_observed_at: None, + after_event_id: None, + limit: None, + }; + assert!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("expired rows hidden before cleanup") + .records + .is_empty() + ); + for expected in [100, 100, 5, 0] { + assert_eq!( + prune_statistics(store) + .await + .expect("global bounded expiration"), + expected + ); + } + let options = StatisticsQuery { + schema: "main".into(), + name: "orders_🧾_000".into(), + ..options + }; + assert_eq!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("retained points survive") + .records + .len(), + 100 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index c83b56f94c..529087b313 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -180,6 +180,12 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp (location, other), ) .await; + crate::api::tests::statistics_helpers::statistics_reads( + store, + credentials, + (location, other, reader_session), + ) + .await; let mut actual = Vec::new(); for _ in 0..2 { let page = read_person_catalog( @@ -303,6 +309,27 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp ) .await .expect("revoke"); + assert!(matches!( + crate::warehouse_metadata::main::read_person_statistics::read_person_statistics( + store, + credentials, + ( + location, + "development", + &crate::warehouse_metadata::models::StatisticsQuery { + database: None, + schema: "main".into(), + name: "orders_🧾_000".into(), + after_observed_at: None, + after_event_id: None, + limit: None, + } + ), + DefaultRole::None, + ) + .await, + Err(ControlError::Denied { .. }) + )); assert!(matches!( crate::warehouse_metadata::main::read_person_node_status::read_person_node_status( store, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 3e5744e2ff..074dad859a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -16,6 +16,12 @@ mod oidc_test_types; mod output_helpers; #[path = "recall/helpers.rs"] mod recall_helpers; +#[path = "statistics/eligibility/helpers.rs"] +mod statistics_eligibility; +#[path = "statistics/helpers.rs"] +mod statistics_helpers; +#[path = "statistics/retention/helpers.rs"] +mod statistics_retention; #[path = "test_catalog_reads.rs"] mod test_catalog_reads; #[path = "test_changes.rs"] @@ -38,6 +44,8 @@ mod test_passwords; mod test_proxy; #[path = "test_reads.rs"] mod test_reads; +#[path = "test_statistics_recovery.rs"] +mod test_statistics_recovery; #[path = "test_types.rs"] mod test_types; #[path = "test_warehouse_metadata.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs new file mode 100644 index 0000000000..35d3f76efa --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs @@ -0,0 +1,82 @@ +//! A reset during optional cleanup cannot tear down the HTTP server. + +use std::time::Duration; + +use serde_json::Value; + +use crate::api::tests::changes_helpers::reset_proxy; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::WorkerRedirectTestCase; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_statistics_cleanup_when_connection_resets_then_http_survives() { + let test_cases = [WorkerRedirectTestCase { + description: "PostgreSQL cleanup TCP reset", + expected_status: reqwest::StatusCode::OK, + }]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(Backend::Postgres) + .await + .expect("isolated store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let ControlStore::Postgres(pool) = store else { + panic!("PostgreSQL case") + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json", + ); + let data: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("statistics fixture"); + let fault = &data["cleanup_interruption"]; + sqlx_core::raw_sql::raw_sql(fault["block_sql"].as_str().expect("block SQL")) + .execute(pool) + .await + .expect("pause actual cleanup"); + let proxy = reset_proxy(store).await; + let api = start_api(&proxy.store).await; + tokio::time::timeout( + Duration::from_secs(6), + sqlx_core::raw_sql::raw_sql(fault["wait_sql"].as_str().expect("wait SQL")) + .execute(pool), + ) + .await + .expect("bounded active cleanup wait") + .expect("actual prune is active"); + proxy.reset.send_modify(|epoch| *epoch += 1); + tokio::time::sleep(Duration::from_millis(100)).await; + sqlx_core::raw_sql::raw_sql(fault["restore_sql"].as_str().expect("restore SQL")) + .execute(pool) + .await + .expect("remove fault"); + let response = reqwest::Client::new() + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("HTTP survives cleanup reset"); + assert_eq!( + response.status(), + test_case.expected_status, + "{}", + test_case.description + ); + let schema: Value = response.json().await.expect("OpenAPI after cleanup reset"); + assert!( + schema["paths"] + .get("/api/control/locations/{location_id}/targets/{target}/statistics") + .is_some() + ); + stop_api(api).await; + close_store(proxy.store).await; + proxy.shutdown.send(()).expect("stop proxy"); + proxy.task.await.expect("proxy stopped"); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 7feba25190..5b9ca8f35c 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,9 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DROP TABLE sqb_warehouse_statistics", + "DROP INDEX sqb_catalog_statistics_scan", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_statistics'", "DROP INDEX sqb_fingerprint_node_latest", "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN node_type", "ALTER TABLE sqb_warehouse_fingerprints DROP COLUMN node_name", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 858bd23966..230b5bbee2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -16,3 +16,4 @@ pub(crate) mod sqlite_cursor; pub(crate) mod state_column_evolution; pub(crate) mod warehouse_catalog; pub(crate) mod warehouse_metadata; +pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 9586f0df14..a5c265e81e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -29,6 +29,7 @@ use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::native_host::warehouse_catalog::read_catalog; +use crate::bindings::_helpers::native_host::warehouse_statistics::read_statistics; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; @@ -41,6 +42,7 @@ pub(crate) fn read_history( request: &WarehouseHistoryRead, ) -> PyResult { let location = match request { + WarehouseHistoryRead::Statistics(read) => &read.location, WarehouseHistoryRead::Catalog(read) => &read.location, WarehouseHistoryRead::SourceFreshness(read) => &read.location, WarehouseHistoryRead::Fingerprints(read) => &read.location, @@ -125,6 +127,9 @@ fn read_page( }; let event_columns: Vec<&str> = event_columns.iter().map(String::as_str).collect(); let (location, after, since, limit, table, columns, timestamp) = match request { + WarehouseHistoryRead::Statistics(read) => { + return read_statistics(py, (adapter, connection), read); + } WarehouseHistoryRead::Catalog(read) => { return read_catalog(py, (adapter, connection), read); } @@ -372,7 +377,7 @@ fn read_page( }; } let observed = match &record { - WarehouseHistoryRecord::Catalog(_) => { + WarehouseHistoryRecord::Catalog(_) | WarehouseHistoryRecord::Statistics(_) => { return Err(PyValueError::new_err( "catalog does not use timestamp history filtering", )); @@ -405,7 +410,7 @@ fn decode_record( (columns, timestamp): (&[&str], &str), ) -> PyResult { let (location, record) = match request { - WarehouseHistoryRead::Catalog(_) => { + WarehouseHistoryRead::Catalog(_) | WarehouseHistoryRead::Statistics(_) => { return Err(PyValueError::new_err( "catalog does not use state-table row decoding", )); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs new file mode 100644 index 0000000000..3114d4e852 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs @@ -0,0 +1,61 @@ +//! Metadata estimates never execute exact counts or imply source-data currency. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + StatisticsRead, WarehouseHistoryPage, WarehouseHistoryRecord, WarehouseStatisticsRecord, +}; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::records::fields::field; + +pub(crate) fn read_statistics( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &StatisticsRead, +) -> PyResult { + if request.names.len() > MAX_HISTORY_RECORDS + || request.names.iter().any(|name| name.trim().is_empty()) + { + return Err(PyValueError::new_err( + "statistics sampling needs at most 100 exact relation names", + )); + } + let mut records: Vec = Vec::new(); + for name in &request.names { + let estimate = call_with( + &adapter.getattr("estimate_relation_row_count")?, + &[ + ("connection", connection.clone()), + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schema", + request.location.schema.clone().into_bound_py_any(py)?, + ), + ("name", name.into_bound_py_any(py)?), + ("metadata_only", true.into_bound_py_any(py)?), + ], + )?; + records.push(WarehouseHistoryRecord::Statistics( + WarehouseStatisticsRecord { + state_database: request.location.database.clone(), + state_schema: request.location.schema.clone(), + name: name.clone(), + observed_at: chrono::Utc::now(), + row_count: field(&estimate, "row_count")?.extract()?, + detail: field(&estimate, "detail")?.extract()?, + byte_count: field(&estimate, "byte_count")?.extract()?, + byte_count_basis: field(&estimate, "byte_count_basis")?.extract()?, + }, + )); + } + Ok(WarehouseHistoryPage { + records, + next_after: request.next_after.map(|id| vec![Some(id.to_string())]), + }) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 56ccde09be..16bc3178b1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -237,9 +237,16 @@ class CatalogInspectionTestCase: connection_key: str expected_table_type: str expected_view_type: str + expected_metadata_rows: int | None = None @dataclass(frozen=True) class CatalogBoundaryTestCase: description: str expected_page_delivered: bool + + +@dataclass(frozen=True) +class StatisticsRecoveryTestCase: + description: str + expected_catalog_presence: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 51d7aa990e..abded396bf 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -35,3 +35,23 @@ def catalog_facts(control: Path) -> list[dict[str, Any]]: "AND json_extract(payload, '$.record.name')='orders_🧾' ORDER BY storage_id" ).fetchall() return [json.loads(row[0]) for row in rows] + + +def wait_for_statistics( + control: Path, name: str, process: Popen[bytes], log: Path +) -> list[dict[str, Any]]: + """Observe actual metadata-only sampling through the installed location and HTTP ingestion.""" + deadline: float = time.monotonic() + 45 + records: list[dict[str, Any]] = [] + while not records: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with closing(sqlite3.connect(control)) as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT record_json FROM sqb_warehouse_statistics " + "WHERE relation_name=? ORDER BY observed_at,event_id", + (name,), + ).fetchall() + records = [json.loads(row[0]) for row in rows] + time.sleep(0.05) + return records diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index c90f2b8462..2b5c264dee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -127,6 +127,33 @@ def wait_for_person_catalog( "status": "unknown", "reason": "consumed-input provenance is not recorded", } + statistics_path: str = ( + f"/api/control/locations/{api.location_id}/targets/development/statistics" + "?schema=main&name=orders" + ) + statistics: dict[str, Any] = person_read(api, statistics_path, session=session) + while not statistics["records"]: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (statistics, log.read_text()) + time.sleep(0.05) + statistics = person_read(api, statistics_path, session=session) + sample: dict[str, Any] = statistics["records"][0] + assert sample["event_id"] and sample["record"]["observed_at"] + assert sample["record"] | {"observed_at": None} == { + "state_database": None, + "state_schema": "main", + "name": "orders", + "observed_at": None, + "row_count": 1, + "detail": None, + "byte_count": None, + "byte_count_basis": None, + } + assert statistics["next_after"] is None + assert person_read(api, statistics_path + "&database=unlisted", session=session) == { + "records": [], + "next_after": None, + } def verify_security_audit( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py new file mode 100644 index 0000000000..6cf373e8cc --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py @@ -0,0 +1,101 @@ +"""A failed sample must leave catalog reconciliation able to remove dropped relations.""" + +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + StatisticsRecoveryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + catalog_facts, + wait_for_catalog, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + open_legacy_warehouse, + snapshot_api, + wait_for_generation, +) + + +@pytest.mark.parametrize( + "test_case", + [StatisticsRecoveryTestCase("Drop after discovery before sampling", (True, False))], + ids=lambda case: case.description, +) +def test_given_relation_dropped_during_sampling_when_read_fails_then_catalog_can_reconcile( + test_case: StatisticsRecoveryTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json").read_text() + )["drop_during_sampling"] + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, "sqlite", "path") + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + (project / "models").mkdir() + (project / "models/orders.sql").write_text(fixture["model"]) + hook: Path = tmp_path / "sampling-hook" + hook.mkdir() + dropped: Path = hook / "dropped" + (hook / "sitecustomize.py").write_text( + "from pathlib import Path\n" + "import sqlite3\n" + "from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter\n" + "original = SqliteAdapter.estimate_relation_row_count\n" + "def sample(self, *, connection, database, schema, name, metadata_only=False):\n" + f" if name == {fixture['name']!r} and metadata_only:\n" + f" marker = Path({str(dropped)!r})\n" + " if not marker.exists():\n" + f" with sqlite3.connect({str(project / 'warehouse.db')!r}) as writer:\n" + f" writer.execute({fixture['drop_sql']!r})\n" + " marker.write_text('dropped after discovery')\n" + f" raise RuntimeError({fixture['failure']!r})\n" + " return original(self, connection=connection, database=database, schema=schema, " + "name=name, metadata_only=metadata_only)\n" + "SqliteAdapter.estimate_relation_row_count = sample\n" + ) + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "PYTHONPATH": str(hook)}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + deadline: float = time.monotonic() + 30 + while not dropped.exists(): + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.05) + wait_for_catalog(control, [], location, log) + assert ( + tuple(fact["present"] for fact in catalog_facts(control)) + == test_case.expected_catalog_presence + ) + assert "Warehouse history read failed" in log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py index 1ca69c726f..ec358d377c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py @@ -7,6 +7,7 @@ import subprocess import sys import time +from collections import Counter from contextlib import closing from pathlib import Path from typing import Any @@ -23,6 +24,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( catalog_facts, wait_for_catalog, + wait_for_statistics, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( location_process, @@ -170,7 +172,7 @@ def test_given_non_authoritative_adapter_scope_when_reading_catalog_then_no_page "test_case", [ CatalogInspectionTestCase( - "DuckDB automatic catalog", "duckdb", "database", "BASE TABLE", "VIEW" + "DuckDB automatic catalog", "duckdb", "database", "BASE TABLE", "VIEW", 0 ), CatalogInspectionTestCase( "SQLite automatic catalog", "sqlite", "path", "BASE TABLE", "VIEW" @@ -220,6 +222,14 @@ def test_given_served_location_when_catalog_changes_then_facts_and_current_proje ): _ = wait_for_generation(api, "development", 1, location, log) wait_for_catalog(control, [expected], location, log) + samples: list[dict[str, Any]] = wait_for_statistics( + control, expected["name"], location, log + ) + assert len(samples) == 1 + assert samples[0]["name"] == expected["name"] + assert samples[0]["byte_count"] is None + assert samples[0]["byte_count_basis"] is None + assert samples[0]["row_count"] == test_case.expected_metadata_rows assert [fact["present"] for fact in catalog_facts(control)] == [True] time.sleep(6) assert [fact["present"] for fact in catalog_facts(control)] == [True] @@ -257,3 +267,13 @@ def test_given_served_location_when_catalog_changes_then_facts_and_current_proje database.commit() wait_for_catalog(control, [expected], location, log) assert catalog_facts(control)[-1]["record"] == expected + current: list[dict[str, Any]] = wait_for_statistics( + control, expected["name"], location, log + ) + assert samples[0] in current + assert ( + Counter(record["observed_at"][:13] for record in current)[ + samples[0]["observed_at"][:13] + ] + == 1 + ) From 98619dd377d8731f4bbf20cbf41bb3c5d8579100 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:39:03 +0000 Subject: [PATCH 106/138] fix: pin discovered Python execution source in snapshots Include task, asset and check source in immutable node content so workers reject edits before executing models. Retain source locations rather than additional callables or source reads during ordinary compilation. Cover unchanged and edited attached checks through real scheduled workers, plus relocatable task, asset and check snapshots using DuckShop fixtures. --- .../python_execution_inputs.json | 6 +++ .../compile_driver/assembly_project.rs | 24 ++++++++- .../_helpers/compile_records/constants.rs | 1 + .../compiled_project/snapshot_boundary.rs | 33 +++++++----- .../_helpers/compiled_views/getters.rs | 1 + .../_helpers/compiled_views/project_view.rs | 15 ++++++ src/sqlbuild/_native.pyi | 4 ++ src/sqlbuild/compiler/compile/types.py | 2 + .../publication/test_worker.py | 54 +++++++++++++++++++ .../test_orchestration_snapshot.py | 13 +++++ 10 files changed, 140 insertions(+), 13 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json new file mode 100644 index 0000000000..87f831d781 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json @@ -0,0 +1,6 @@ +{ + "initial": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n return ctx.pass_()\n", + "changed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n raise RuntimeError('changed check executed after publication')\n", + "task": "from sqlbuild.tasks import task\n@task(description='Notify support.')\ndef notify_support(ctx):\n return None\n", + "asset": "from sqlbuild.assets import asset\n@asset(description='Order export.')\ndef order_export(ctx):\n return []\n" +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index f9178da352..9f47662260 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -1,5 +1,7 @@ //! Compiled project assembly: the project's models, resources, checks and semantic diagnostics. +use std::path::PathBuf; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; @@ -29,7 +31,7 @@ use crate::bindings::_helpers::model_config::template_data::early_model_template use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_get; +use crate::bindings::_helpers::records::fields::{mapping_get, path_posix}; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; @@ -287,6 +289,26 @@ fn project_fields<'py>( for name in ["loader_functions", "hook_functions"] { fields.set_item(name, native_field(&discovered, name)?)?; } + let python_inputs: Bound<'py, PyDict> = PyDict::new(py); + for (kind, collection) in [ + ("task", "task_functions"), + ("asset", "asset_functions"), + ("check", "check_functions"), + ] { + for function in native_field(&discovered, collection)?.try_iter()? { + let function = function?; + let name: String = native_field(&function, "name")?.extract()?; + let path: PathBuf = native_field(&function, "file_path")?.extract()?; + python_inputs.set_item( + format!("{kind}:{name}"), + ( + path_posix(&native_field(&function, "relative_path")?)?, + path, + ), + )?; + } + } + fields.set_item("python_execution_inputs", python_inputs)?; fields.set_item( "enforce_explicit_references", native_field(&project_config, "references")?.getattr("enforce_explicit")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 33cf4c8205..15329c6c5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1590,6 +1590,7 @@ const R_COMPILED_PROJECT: RecordSpec = RecordSpec { compared: false, shown: false, }, + FieldSpec::new("python_execution_inputs", FieldDefault::EmptyDict), ], }; const R_COMPILED_SQL_TEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs index 8eaa09494a..e6093663b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs @@ -5,6 +5,7 @@ use std::fs::File; use std::io::Read; use std::path::PathBuf; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use serde_json::{Map, Value, json}; @@ -126,6 +127,8 @@ pub(crate) fn boundary_nodes( )?; } } + let mut python_sources: Vec<(String, (String, PathBuf))> = + project.python_execution_inputs().into_iter().collect(); for (kind, objects) in [ ("loader", project.loader_functions(py)), ("hook", project.hook_functions(py)), @@ -135,20 +138,26 @@ pub(crate) fn boundary_nodes( let item = item?; let name: String = field(&item, "name")?.extract()?; let path: PathBuf = field(&item, "file_path")?.extract()?; - let source = py.detach(|| std::fs::read_to_string(path))?; - nodes = insert_node( - nodes, - format!("{kind}:{name}"), - SnapshotNode { - kind: kind.into(), - definition: json!({"name": name, "path": relative_path(&item)?}), - raw_sql: None, - compiled_sql: None, - python_source: Some(source), - }, - )?; + python_sources.push((format!("{kind}:{name}"), (relative_path(&item)?, path))); } } + for (key, (relative, path)) in python_sources { + let (kind, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("invalid Python execution input identity"))?; + let source = py.detach(|| std::fs::read_to_string(path))?; + nodes = insert_node( + nodes, + key.clone(), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "path": relative}), + raw_sql: None, + compiled_sql: None, + python_source: Some(source), + }, + )?; + } for item in project.sql_hook_files(py).bind(py).try_iter()? { let item = item?; let name: String = field(&item, "name")?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs index dda648c730..c28a376f1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs @@ -635,6 +635,7 @@ fn read_native_project_view<'py>( "sql_lexical_syntax" => this.sql_lexical_syntax(py).into_bound_py_any(py), "compile_cache_dir" => this.compile_cache_dir(py).into_bound_py_any(py), "loader_functions" => this.loader_functions(py).into_bound_py_any(py), + "python_execution_inputs" => this.python_execution_inputs().into_bound_py_any(py), "hook_functions" => this.hook_functions(py).into_bound_py_any(py), "sql_hook_files" => this.sql_hook_files(py).into_bound_py_any(py), "materialization_files" => this.materialization_files(py).into_bound_py_any(py), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs index 90cdf45297..f28724aab8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs @@ -1,5 +1,8 @@ //! A compiled project, read from the native compiled project that owns its models. +use std::collections::BTreeMap; +use std::path::PathBuf; + use pyo3::exceptions::PyTypeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; @@ -46,6 +49,8 @@ struct BoundaryInputs { compile_cache_dir: Py, /// Discovered loader functions, user callables. loader_functions: Py, + /// Task/asset/check source locations, without retaining additional Python callables. + python_execution_inputs: BTreeMap, /// Discovered Python hooks, user callables. hook_functions: Py, /// Discovered SQL hook files. @@ -123,6 +128,7 @@ impl ProjectFields { sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), loader_functions: required(fields, "loader_functions")?.unbind(), + python_execution_inputs: required(fields, "python_execution_inputs")?.extract()?, hook_functions: required(fields, "hook_functions")?.unbind(), sql_hook_files: required(fields, "sql_hook_files")?.unbind(), materialization_files: required(fields, "materialization_files")?.unbind(), @@ -195,6 +201,10 @@ impl ProjectFields { self.boundary.compile_cache_dir.bind(py), )?; fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; + fields.set_item( + "python_execution_inputs", + &self.boundary.python_execution_inputs, + )?; fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; fields.set_item( @@ -391,6 +401,11 @@ impl NativeProjectView { self.fields.boundary.loader_functions.clone_ref(py) } + #[getter] + pub(crate) fn python_execution_inputs(&self) -> BTreeMap { + self.fields.boundary.python_execution_inputs.clone() + } + #[getter] pub(crate) fn hook_functions(&self, py: Python<'_>) -> Py { self.fields.boundary.hook_functions.clone_ref(py) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 93e5d60093..42a4056e4b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1823,6 +1823,8 @@ class NativeProjectView: @property def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: ... @property + def python_execution_inputs(self) -> dict[str, tuple[str, Path]]: ... + @property def hook_functions(self) -> tuple[DiscoveredHookFunction, ...]: ... @property def enforce_explicit_references(self) -> bool: ... @@ -15615,6 +15617,7 @@ class CompiledProject(NativeRecord): sql_expansions: dict[Path, CompiledSqlExpansion] native_session: Any | None native_project: NativeCompiledProject | None + python_execution_inputs: dict[str, tuple[str, Path]] def __init__( self, run_id: str, @@ -15651,6 +15654,7 @@ class CompiledProject(NativeRecord): sql_expansions: dict[Path, CompiledSqlExpansion] = ..., native_session: Any | None = ..., native_project: NativeCompiledProject | None = ..., + python_execution_inputs: dict[str, tuple[str, Path]] = ..., ) -> None: ... @property def lineage_graph(self) -> NativeProjectGraph: ... diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index a093f3faa9..d55f426473 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -254,6 +254,8 @@ def sql_scenarios(self) -> tuple[CompiledSqlScenario, ...]: ... @property def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: ... @property + def python_execution_inputs(self) -> dict[str, tuple[str, Path]]: ... + @property def hook_functions(self) -> tuple[DiscoveredHookFunction, ...]: ... @property def enforce_explicit_references(self) -> bool: ... diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 7e2793f754..042e0f2c49 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -487,6 +487,60 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "unchanged attached Python check", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "success", + [(17,)], + files={"python/checks/orders.py": "initial"}, + ), + WorkerExecutionTestCase( + "attached Python check changed after publication", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "failed", + [], + files={"python/checks/orders.py": "initial"}, + files_after_publication={"python/checks/orders.py": "changed"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_attached_python_code_when_a_worker_starts_then_publication_pins_it_before_execution( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json" + ).read_text() + ) + test_case = replace( + test_case, + files={path: fixture[key] for path, key in test_case.files.items()}, + files_after_publication={ + path: fixture[key] for path, key in test_case.files_after_publication.items() + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + nodes: list[tuple[str, str | None]] = control.execute( + "SELECT node_key,started_at FROM sqb_run_nodes WHERE run_id=? ORDER BY node_key", + (run["run_id"],), + ).fetchall() + assert [(name, started is not None) for name, started in nodes] == [ + ("model:orders", test_case.expected_status == "success") + ] + assert (worker.project / "warehouse.duckdb").exists() == ( + test_case.expected_status == "success" + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 0d7a1e564f..4bc4967f34 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -100,6 +100,19 @@ }, frozenset({"model", "hook"}), ), + SnapshotCatalogTestCase( + "DuckShop Python execution inputs", + TYPE_PROOF_FILES + | { + f"python/{kind}s/orders.py": json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json" + ).read_text() + )[source] + for kind, source in (("task", "task"), ("asset", "asset"), ("check", "initial")) + }, + frozenset({"model", "task", "asset", "check"}), + ), SnapshotCatalogTestCase( "SQL hooks", TYPE_PROOF_FILES From f6e2dd51bd8fec04a618614934da174c596f7ca8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:45:14 +0000 Subject: [PATCH 107/138] feat: publish observational attached-check lifecycle facts Report claim-bound actual check starts and outcomes without expanding frozen runnable membership, write claims or node counters. Feed existing sensors from check observations and preserve decisive independent health. Refresh cancellation before classifying a failed check result. Cover heartbeat-only driver cancellation, real outcome workers, failure-triggered inventory execution, and SQLite/PostgreSQL authority and history boundaries with synthetic fixtures. --- .../orchestration/attached_checks.json | 8 + clients/typescript/src/schema.d.ts | 9 + .../src/runs/_helpers/lifecycle/changes.rs | 9 +- .../src/runs/_helpers/lifecycle/checks.rs | 54 ++++ .../src/runs/_helpers/lifecycle/mod.rs | 1 + .../src/runs/_helpers/lifecycle/projection.rs | 4 + .../sqlbuild-orchestration/src/runs/models.rs | 20 ++ .../src/sensors/_helpers/nodes.rs | 22 +- .../src/sensors/_helpers/transitions.rs | 5 +- .../src/store/_helpers/migration.rs | 1 + .../src/store/check_observations.sql | 10 + .../src/store/main/migrate_store.rs | 3 +- .../tests/runs/test_checks.rs | 259 ++++++++++++++++++ .../tests/runs/test_contracts.rs | 2 + .../tests/store/helpers.rs | 4 + .../_helpers/build_command/execution.rs | 16 +- .../build_progress/build_callbacks.rs | 36 +++ .../compiled_project/snapshot_declarations.rs | 2 +- .../compiled_project/snapshot_execution.rs | 82 ++++++ .../_helpers/native_host/constants.rs | 2 + .../native_host/orchestration_triggers.rs | 53 +++- src/sqlbuild/_native.pyi | 2 + .../python_nodes/_helpers/python_checks.py | 2 + src/sqlbuild/executor/python_nodes/models.py | 1 + .../publication/_test_types.py | 6 + .../publication/test_worker.py | 163 +++++++++++ 26 files changed, 744 insertions(+), 32 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/attached_checks.json create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs create mode 100644 crates/sqlbuild-orchestration/src/store/check_observations.sql create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_checks.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json new file mode 100644 index 0000000000..d29390f156 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -0,0 +1,8 @@ +{ + "passed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.pass_()\n", + "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", + "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", + "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('SELECT sum(i) FROM range(10000000000) AS t(i)')\n return ctx.pass_()\n", + "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", + "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 406e26850e..17d1493129 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1253,6 +1253,15 @@ export interface components { /** Format: int64 */ rows_affected?: number | null; skipped_because_of?: string | null; + } | { + /** @enum {string} */ + kind: "check_started"; + node_key: string; + } | { + /** @enum {string} */ + kind: "check_finished"; + node_key: string; + outcome: components["schemas"]["NodeOutcome"]; } | { /** @enum {string} */ kind: "unresponsive"; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index f6a099cf26..c71085d635 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -68,6 +68,8 @@ pub(crate) fn validate_authority( | RunChange::Started | RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } ), RunAuthority::Worker { run_id, claim_id } => { run_id == request.run_id @@ -77,6 +79,8 @@ pub(crate) fn validate_authority( RunChange::Started | RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } | RunChange::Recovered | RunChange::Finished { outcome: RunOutcome::Success @@ -123,7 +127,10 @@ pub(crate) fn validate_change( RunChange::Started => { state.worker_instance.is_some() && matches!(state.status, Queued | Launched) } - RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } => { matches!(state.status, Running | Unresponsive) } RunChange::Unresponsive => state.status == Running, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs new file mode 100644 index 0000000000..1c1dc52a7e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs @@ -0,0 +1,54 @@ +//! Attached checks are immutable observations, not additions to the frozen execution selection. + +use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn check_execution_history( + transaction: ControlTransaction, + request: &RunChangeRequest, +) -> Result { + let (node_key, outcome) = match &request.change { + RunChange::CheckStarted { node_key } => (node_key, None), + RunChange::CheckFinished { node_key, outcome } => (node_key, Some(*outcome)), + _ => return Err(ControlError::Invalid("check fact required".into())), + }; + let (transaction, nodes) = query(transaction, + "SELECT n.node_key FROM sqb_runs r JOIN sqb_snapshot_nodes n ON n.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND n.node_key=$2 AND n.node_kind='check' LIMIT 1", + vec![Parameter::Id(request.run_id), Parameter::Text(node_key.clone())]).await?; + if !node_key.starts_with("check:") || nodes.is_empty() { + return Err(ControlError::Invalid( + "check is outside the frozen snapshot".into(), + )); + } + let (transaction, previous) = query( + transaction, + "SELECT event_type FROM recall_events WHERE invocation_id=$1 AND producer='scheduler' \ + AND run_id=$2 AND event_type IN ('run_check_started','run_check_finished') \ + AND (payload -> 'change' -> 'change' ->> 'node_key')=$3 \ + AND (payload -> 'change' ->> 'claim_id')=$4 AND event_id<>$5 \ + ORDER BY storage_id DESC LIMIT 1", + vec![ + Parameter::Text(format!("scheduler/{}", request.run_id)), + Parameter::Text(request.run_id.to_string()), + Parameter::Text(node_key.clone()), + Parameter::OptionalText(request.claim_id.map(|id| id.to_string())), + Parameter::Text(request.event_id.to_string()), + ], + ) + .await?; + let previous = previous.first().and_then(|row| row["event_type"].as_str()); + let valid = match outcome { + None => previous.is_none(), + Some(NodeOutcome::Skipped) => previous.is_none() || previous == Some("run_check_started"), + Some(_) => previous == Some("run_check_started"), + }; + if !valid { + return Err(ControlError::Invalid( + "check fact is inconsistent with its execution history".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index b63e6414ba..1d0d31e732 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -1,5 +1,6 @@ pub(in crate::runs) mod change_run; pub(in crate::runs) mod changes; +pub(in crate::runs) mod checks; pub(in crate::runs) mod claims; pub(in crate::runs) mod events; pub(in crate::runs) mod nodes; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index b430f35721..a386ad067b 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use serde_json::json; +use crate::runs::_helpers::lifecycle::checks::check_execution_history; use crate::runs::_helpers::lifecycle::nodes::project_node; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; @@ -18,6 +19,9 @@ pub(crate) async fn project_change( let now = transaction.now; match &request.change { RunChange::LaunchFailed { .. } => {} + RunChange::CheckStarted { .. } | RunChange::CheckFinished { .. } => { + return check_execution_history(transaction, request).await; + } RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { return project_node(transaction, request, &state.node_counts_json).await; } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index a2f946e250..a6b13e74ee 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -274,6 +274,14 @@ pub enum RunChange { error_code: Option, skipped_because_of: Option, }, + /// Observational attached-check facts never expand the frozen execution selection. + CheckStarted { + node_key: String, + }, + CheckFinished { + node_key: String, + outcome: NodeOutcome, + }, Unresponsive, Recovered, CancelRequested { @@ -296,12 +304,24 @@ impl RunChange { Self::Started => "run_started", Self::NodeStarted { .. } => "run_node_started", Self::NodeFinished { .. } => "run_node_finished", + Self::CheckStarted { .. } => "run_check_started", + Self::CheckFinished { .. } => "run_check_finished", Self::Unresponsive => "run_unresponsive", Self::Recovered => "run_recovered", Self::CancelRequested { .. } => "run_cancel_requested", Self::Finished { .. } => "run_terminated", } } + + pub(crate) fn finished_node(&self) -> Option<(&str, NodeOutcome)> { + match self { + Self::NodeFinished { + node_key, outcome, .. + } + | Self::CheckFinished { node_key, outcome } => Some((node_key, *outcome)), + _ => None, + } + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs index b556219f83..0ec532f471 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -30,7 +30,7 @@ pub(crate) async fn consume_node_events( let (mut transaction, rows) = query( transaction, "SELECT storage_id,event_id,run_id,payload FROM recall_events \ - WHERE producer='scheduler' AND event_type='run_node_finished' \ + WHERE producer='scheduler' AND event_type IN ('run_node_finished','run_check_finished') \ AND invocation_id='scheduler/' || run_id AND storage_id>$1 \ ORDER BY storage_id LIMIT 100", vec![Parameter::Integer(evaluation.cursor.event_storage_id)], @@ -46,10 +46,7 @@ pub(crate) async fn consume_node_events( if request.run_id != event.run_id || request.event_id != event.event_id { return Err(ControlError::Invalid("node event identity mismatch".into())); } - let RunChange::NodeFinished { - node_key, outcome, .. - } = &request.change - else { + let Some((node_key, outcome)) = request.change.finished_node() else { continue; }; if !selection.contains(node_key) @@ -67,7 +64,7 @@ pub(crate) async fn consume_node_events( (transaction, matches) = matches_transition( transaction, evaluation, - (node_key, *outcome, event.storage_id), + (node_key, outcome, event.storage_id), kinds, ) .await?; @@ -249,16 +246,20 @@ async fn prior_start( request: &RunChangeRequest, storage_id: i64, ) -> Result<(ControlTransaction, bool), ControlError> { - let (RunChange::NodeFinished { node_key, .. }, Some(claim_id)) = - (&request.change, request.claim_id) + let (Some((node_key, _)), Some(claim_id)) = (request.change.finished_node(), request.claim_id) else { return Ok((transaction, false)); }; + let started_type = if matches!(request.change, RunChange::CheckFinished { .. }) { + "run_check_started" + } else { + "run_node_started" + }; let (transaction, rows) = query( transaction, "SELECT s.event_id FROM sqb_runs r JOIN recall_events s ON s.run_id=CAST(r.run_id AS TEXT) \ WHERE s.invocation_id=$1 AND s.run_id=$2 AND s.producer='scheduler' \ - AND s.event_type='run_node_started' AND s.storage_id<$3 \ + AND s.event_type=$9 AND s.storage_id<$3 \ AND (s.payload -> 'change' ->> 'claim_id')=$4 \ AND (s.payload -> 'change' -> 'change' ->> 'node_key')=$5 \ AND r.run_id=$6 AND r.location_id=$7 AND r.target=$8 AND r.kind<>'group' LIMIT 1", @@ -267,10 +268,11 @@ async fn prior_start( Parameter::Text(request.run_id.to_string()), Parameter::Integer(storage_id), Parameter::Text(claim_id.to_string()), - Parameter::Text(node_key.clone()), + Parameter::Text(node_key.into()), Parameter::Id(request.run_id), Parameter::Id(evaluation.location_id), Parameter::Text(evaluation.target.clone()), + Parameter::Text(started_type.into()), ], ) .await?; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs index 5c9becb4e4..3c0e22957c 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs @@ -29,14 +29,15 @@ pub(crate) async fn matches_transition( transaction, "SELECT e.payload -> 'change' -> 'change' ->> 'outcome' AS outcome \ FROM recall_events e JOIN sqb_runs r ON e.run_id=CAST(r.run_id AS TEXT) \ - WHERE e.producer='scheduler' AND e.event_type='run_node_finished' \ + WHERE e.producer='scheduler' AND e.event_type IN ('run_node_finished','run_check_finished') \ AND e.invocation_id='scheduler/' || e.run_id AND e.storage_id<$1 \ AND (e.payload -> 'change' -> 'change' ->> 'node_key')=$2 \ AND (e.payload -> 'change' -> 'change' ->> 'outcome') IN ('success','failed') \ AND r.location_id=$3 AND r.target=$4 AND r.kind<>'group' \ AND EXISTS (SELECT 1 FROM recall_events s \ WHERE s.invocation_id=e.invocation_id AND s.run_id=e.run_id \ - AND s.producer='scheduler' AND s.event_type='run_node_started' \ + AND s.producer='scheduler' AND s.event_type=CASE WHEN e.event_type='run_check_finished' \ + THEN 'run_check_started' ELSE 'run_node_started' END \ AND s.storage_id 'change' ->> 'claim_id')=(e.payload -> 'change' ->> 'claim_id') \ AND (s.payload -> 'change' -> 'change' ->> 'node_key')=$2) \ diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 7715b48e4c..a270a12be8 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -37,6 +37,7 @@ pub(crate) const SECURITY_AUDIT_SINK: &str = include_str!("../security_audit_sin pub(crate) const WAREHOUSE_CATALOG: &str = include_str!("../warehouse_catalog.sql"); pub(crate) const FINGERPRINT_LOOKUP: &str = include_str!("../fingerprint_lookup.sql"); pub(crate) const WAREHOUSE_STATISTICS: &str = include_str!("../warehouse_statistics.sql"); +pub(crate) const CHECK_OBSERVATIONS: &str = include_str!("../check_observations.sql"); pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/check_observations.sql b/crates/sqlbuild-orchestration/src/store/check_observations.sql new file mode 100644 index 0000000000..842b50c413 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/check_observations.sql @@ -0,0 +1,10 @@ +CREATE INDEX sqb_observed_node_cursor ON recall_events(storage_id) +WHERE producer='scheduler' AND event_type IN ('run_node_finished','run_check_finished'); +CREATE INDEX sqb_observed_node_identity ON recall_events( + invocation_id, (payload -> 'change' -> 'change' ->> 'node_key'), + (payload -> 'change' ->> 'claim_id'), storage_id DESC +) WHERE producer='scheduler'; +CREATE INDEX sqb_observed_node_decisive ON recall_events( + (payload -> 'change' -> 'change' ->> 'node_key'), storage_id DESC +) WHERE producer='scheduler' AND event_type IN ('run_node_finished','run_check_finished') +AND (payload -> 'change' -> 'change' ->> 'outcome') IN ('success','failed'); diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 54e98409db..12150ca8ed 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -4,7 +4,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, + ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, CHECK_OBSERVATIONS, DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, FINGERPRINT_LOOKUP, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, @@ -65,6 +65,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ("warehouse_catalog", WAREHOUSE_CATALOG), ("fingerprint_lookup", FINGERPRINT_LOOKUP), ("warehouse_statistics", WAREHOUSE_STATISTICS), + ("check_observations", CHECK_OBSERVATIONS), ]; let mut digests = vec![digest.clone()]; digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs new file mode 100644 index 0000000000..0fe7151327 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs @@ -0,0 +1,259 @@ +//! Check observations are claim-bound facts, never implicit runnable selection members. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_run_nodes::list_run_nodes, + list_runs::list_runs, +}; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::RunLifecycleTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membership_and_execution_history_are_preserved() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite check observations", + backend: Backend::Sqlite, + expected_status: RunStatus::Failed, + }, + RunLifecycleTestCase { + description: "PostgreSQL check observations", + backend: Backend::Postgres, + expected_status: RunStatus::Failed, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + for name in [ + "failed", + "warned", + "skipped", + "cancelled", + "passed", + "blocked", + ] { + input.nodes.insert( + format!("check:{name}"), + SnapshotNode { + kind: "check".into(), + definition: json!({"name": name}), + raw_sql: None, + compiled_sql: None, + python_source: Some("def orders_valid(ctx): return ctx.pass_()".into()), + }, + ); + } + input + .nodes + .insert("check:products".into(), input.nodes["products"].clone()); + let published = publish_snapshot(store, &input).await.expect("snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue( + store, + &request((location, published.snapshot_id), 1, 1, &["orders"]), + ) + .await; + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let report = |change| RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }; + change_run(store, worker, &report(RunChange::Started)) + .await + .expect("started"); + let before = read_recall_events(store, 0, 100) + .await + .expect("before rejected reports"); + for change in [ + RunChange::CheckFinished { + node_key: "check:passed".into(), + outcome: NodeOutcome::Success, + }, + RunChange::CheckStarted { + node_key: "check:unknown".into(), + }, + RunChange::CheckStarted { + node_key: "check:products".into(), + }, + RunChange::CheckStarted { + node_key: "orders".into(), + }, + ] { + assert!(change_run(store, worker, &report(change)).await.is_err()); + } + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rejected facts rolled back"), + before + ); + for (name, outcome) in [ + ("failed", NodeOutcome::Failed), + ("warned", NodeOutcome::Warn), + ("skipped", NodeOutcome::Skipped), + ("cancelled", NodeOutcome::Cancelled), + ("passed", NodeOutcome::Success), + ] { + let started = report(RunChange::CheckStarted { + node_key: format!("check:{name}"), + }); + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &started) + .await + .is_err() + ); + assert!( + change_run( + store, + worker, + &RunChangeRequest { + claim_id: Some(Uuid::now_v7()), + ..started.clone() + } + ) + .await + .is_err() + ); + assert!( + change_run(store, worker, &started) + .await + .expect("actual check start") + ); + assert!( + !change_run(store, worker, &started) + .await + .expect("idempotent start") + ); + let finished = report(RunChange::CheckFinished { + node_key: format!("check:{name}"), + outcome, + }); + assert!( + change_run(store, worker, &finished) + .await + .expect("actual check result") + ); + assert!( + !change_run(store, worker, &finished) + .await + .expect("idempotent completion") + ); + assert!( + change_run(store, worker, &report(finished.change.clone())) + .await + .is_err() + ); + assert!( + change_run(store, worker, &report(started.change.clone())) + .await + .is_err() + ); + } + change_run( + store, + worker, + &report(RunChange::CheckFinished { + node_key: "check:blocked".into(), + outcome: NodeOutcome::Skipped, + }), + ) + .await + .expect("blocked check was not executed"); + let nodes = list_run_nodes(store, claim.request.run_id) + .await + .expect("frozen nodes"); + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0].node_key, "orders"); + assert_eq!(nodes[0].status, "pending"); + let facts = read_recall_events(store, 0, 100) + .await + .expect("check facts"); + assert_eq!( + facts + .iter() + .filter(|row| row["event_type"] == "run_check_started") + .count(), + 5 + ); + assert_eq!( + facts + .iter() + .filter(|row| row["event_type"] == "run_check_finished") + .count(), + 6 + ); + for change in [ + RunChange::NodeStarted { + node_key: "orders".into(), + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + }, + RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "Attached check failed".into(), + exit_code: Some(1), + }, + ] { + change_run(store, worker, &report(change)) + .await + .expect("finish execution without new membership"); + } + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("runs"); + assert_eq!( + runs[0].status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + runs[0].node_counts_json, + BTreeMap::from([("success".into(), 1)]) + ); + assert_eq!(runs[0].selection_json, claim.request.selection); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index d4350c8344..ddc82ea434 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -6,6 +6,8 @@ pub(crate) mod helpers; mod monitor_helpers; #[path = "termination/helpers.rs"] pub(crate) mod terminal_helpers; +#[path = "test_checks.rs"] +mod test_checks; #[path = "test_claims.rs"] mod test_claims; #[path = "test_coalesce.rs"] diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 5b9ca8f35c..9893d5268e 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -80,6 +80,10 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DROP INDEX sqb_observed_node_cursor", + "DROP INDEX sqb_observed_node_identity", + "DROP INDEX sqb_observed_node_decisive", + "DELETE FROM sqb_schema_migrations WHERE name='check_observations'", "DROP TABLE sqb_warehouse_statistics", "DROP INDEX sqb_catalog_statistics_scan", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_statistics'", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 41e53c84da..e92e997318 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::build_command::support::{ boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, }; use crate::bindings::_helpers::build_progress::build_callbacks::{ - BuildProgressCallbacks, batch_complete_callback, + BuildProgressCallbacks, batch_complete_callback, check_callback, }; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; @@ -596,10 +596,16 @@ pub(crate) fn run_post_build_python_checks<'py>( )?; let callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::PythonCheckCallbacks)?, - &[( - "on_check_complete", - native_field(&preparation.callbacks, "write_execution_event")?, - )], + &[ + ( + "on_check_start", + check_callback(&preparation.callbacks, false)?, + ), + ( + "on_check_complete", + check_callback(&preparation.callbacks, true)?, + ), + ], )?; call_with( &host::entry(py, ExecutorHost::ExecutePythonChecks)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index 62e6f13740..86948474e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -615,6 +615,20 @@ impl BuildProgressCallbacks { self.write_event(py, result) } + fn on_check_start(&self, py: Python<'_>, name: &str) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.start_check(py, name)?; + } + Ok(()) + } + + fn on_check_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.completed_check(result)?; + } + self.write_event(py, result) + } + /// Close the optional structured execution event stream. fn close(&self, py: Python<'_>) -> PyResult<()> { let (scheduler, query) = { @@ -668,6 +682,28 @@ pub(crate) fn batch_complete_callback<'py>( Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) } +/// Forward a Python check lifecycle callback directly to its native progress owner. +pub(crate) fn check_callback<'py>( + callbacks: &Bound<'py, PyAny>, + completion: bool, +) -> PyResult> { + let owned: Py = callbacks + .downcast::()? + .clone() + .unbind(); + let forward = move |arguments: &Bound<'_, PyTuple>, + _keywords: Option<&Bound<'_, PyDict>>| + -> PyResult<()> { + let value = arguments.get_item(0)?; + if completion { + owned.get().on_check_complete(arguments.py(), &value) + } else { + owned.get().on_check_start(arguments.py(), value.extract()?) + } + }; + Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c04d8a5435..d30b0e4bff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -203,7 +203,7 @@ pub(crate) fn compile_declarations( sensors, } = compile_sensors( py, - (project_dir, input.target.as_str(), graph), + (project_dir, &input, graph), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index a7aa2f894c..ded2b6c2b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -19,6 +19,7 @@ pub(crate) struct WorkerBuild { pub(crate) worker: Py, execution: RunExecution, completed: Mutex>, + checks: Mutex>>, } impl WorkerBuild { @@ -27,6 +28,7 @@ impl WorkerBuild { execution: worker.get().execution(worker.py())?, worker: worker.clone().unbind(), completed: Mutex::default(), + checks: Mutex::default(), }); for key in state.execution.skipped_nodes.keys() { state.complete(worker.py(), key, (NodeOutcome::Skipped, None, None))?; @@ -195,6 +197,67 @@ impl WorkerBuild { Ok(()) } + pub(crate) fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let key = format!("check:{name}"); + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.clone(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key, None); + Ok(()) + } + + /// A completion without our actual-execution start callback is a blocked check, not a failure. + pub(crate) fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let name: String = field(result, "node_name")?.extract()?; + let key = format!("check:{name}"); + let started = self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(&key); + let outcome = if !started { + NodeOutcome::Skipped + } else if field(result, "passed")?.is_truthy()? { + NodeOutcome::Success + } else if field(result, "warned")?.is_truthy()? { + NodeOutcome::Warn + } else if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + }; + self.finish_check(result.py(), &key, outcome) + } + + fn finish_check(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + self.worker.bind(py).get().change( + py, + &format!("check-finished/{key}"), + RunChange::CheckFinished { + node_key: key.into(), + outcome, + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), Some(outcome)); + Ok(()) + } + /// Never call missing selected nodes successful merely because the executor returned zero. pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; @@ -242,6 +305,25 @@ impl WorkerBuild { /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { let outcome = self.outcome(py, exit_code)?; + let unfinished: Vec = self + .checks + .lock() + .map_err(|_| poisoned())? + .iter() + .filter(|(_, outcome)| outcome.is_none()) + .map(|(key, _)| key.clone()) + .collect(); + for key in unfinished { + self.finish_check( + py, + &key, + if outcome == RunOutcome::Cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + )?; + } self.worker.bind(py).get().change( py, "worker-finished", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 13b4431c30..db99dfcfe9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,6 +4,8 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; +pub(crate) const NODE_EVENT_CONDITION: &str = "node_event"; +pub(crate) const CHECK_KIND: &str = "check"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 98afe8125a..608c4d8092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,6 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; @@ -17,12 +17,14 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, + CALENDAR_WINDOW_PARAMETER, CHECK_KIND, NODE_EVENT_CONDITION, SOURCE_NEW_DATA_CONDITION, + SOURCE_STALE_CONDITION, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -256,7 +258,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( py: Python<'_>, - (project_dir, target, graph): (&Path, &str, &ProjectGraph), + (project_dir, input, graph): (&Path, &SnapshotInput, &ProjectGraph), authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -273,7 +275,8 @@ pub(crate) fn compile_sensors( sensor.name ))); } - if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) + { continue; } let condition = match (sensor.when, sensor.function) { @@ -290,11 +293,12 @@ pub(crate) fn compile_sensors( if let Some(select) = when.select { let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; + condition["selection"] = + sensor_nodes(graph, &input.nodes, &selectors, &when.kind)?; } if let Some(selectors) = when.executed_successfully { condition["executed_successfully"] = - sensor_nodes(graph, &selectors, "executed_successfully")?; + sensor_nodes(graph, &input.nodes, &selectors, "executed_successfully")?; } condition } @@ -319,17 +323,42 @@ pub(crate) fn compile_sensors( Ok(definitions) } -fn sensor_nodes(graph: &ProjectGraph, selectors: &[String], field: &str) -> PyResult { +fn sensor_nodes( + graph: &ProjectGraph, + available: &BTreeMap, + selectors: &[String], + field: &str, +) -> PyResult { if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { return Err(PyValueError::new_err(format!( "{field} needs nonempty selectors" ))); } - let nodes: BTreeSet = resolve_selectors(graph, selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); + let mut nodes: BTreeSet = BTreeSet::new(); + let mut sql_selectors: Vec = Vec::new(); + for selector in selectors { + if field == NODE_EVENT_CONDITION && selector.starts_with("check:") { + if !available + .get(selector) + .is_some_and(|node| node.kind == CHECK_KIND) + { + return Err(PyValueError::new_err(format!( + "unknown check node {selector}" + ))); + } + nodes.insert(selector.clone()); + } else { + sql_selectors.push(selector.clone()); + } + } + if !sql_selectors.is_empty() { + nodes.extend( + resolve_selectors(graph, &sql_selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ); + } if nodes.is_empty() { return Err(PyValueError::new_err(format!( "{field} needs nonempty selectors" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 42a4056e4b..8a67d7d699 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5061,6 +5061,8 @@ class BuildProgressCallbacks: def redraw_transient_line(self) -> None: ... def on_node_complete(self, node_result: object) -> None: ... def write_execution_event(self, result: object) -> None: ... + def on_check_start(self, name: str) -> None: ... + def on_check_complete(self, result: PythonCheckExecutionResult) -> None: ... def close(self) -> None: ... class AuditProgressReporter: diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 87027f7f5c..220af44090 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -148,6 +148,8 @@ def _execute_one_python_check( ) as lifecycle, ): if blocked is None: + if callbacks.on_check_start is not None: + callbacks.on_check_start(check_function.name) node: DiscoveredPythonNode = python_graph.nodes_by_name[check_function.name] warnings: list[str] = [] context: CheckContext = CheckContext( diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 4237e12f99..e41a47874f 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -404,3 +404,4 @@ class PythonCheckCallbacks: logger: logging.Logger | None = None identity_recorder: PythonIdentityRecorder | None = None on_check_complete: Callable[[PythonCheckExecutionResult], None] | None = None + on_check_start: Callable[[str], None] | None = None diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 16bc3178b1..b2319ab00d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -113,6 +113,12 @@ class WorkerExecutionTestCase: launcher: Literal["native", "uncooperative", "docker"] = "native" +@dataclass(frozen=True) +class CheckExecutionTestCase(WorkerExecutionTestCase): + description: str + expected_check_outcome: str = "success" + + @dataclass(frozen=True, kw_only=True) class PersonJourneyTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 042e0f2c49..80d8941720 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -27,6 +27,7 @@ TYPE_PROOF_FILES, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CheckExecutionTestCase, ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, @@ -541,6 +542,168 @@ def test_given_attached_python_code_when_a_worker_starts_then_publication_pins_i ) +@pytest.mark.parametrize( + "test_case", + [ + CheckExecutionTestCase( + "passed", "SELECT 17 AS amount", "SELECT amount FROM orders", "success", [(17,)] + ), + CheckExecutionTestCase( + "warned", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "success", + [(17,)], + expected_check_outcome="warn", + ), + CheckExecutionTestCase( + "failed", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "failed", + [(17,)], + expected_check_outcome="failed", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_attached_check_when_a_worker_executes_then_its_own_outcome_is_durable( + test_case: CheckExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/attached_checks.json").read_text() + ) + test_case = replace( + test_case, files={"python/checks/orders.py": fixture[test_case.description]} + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run["selection_json"] == ["model:orders"] + assert run["node_counts_json"] == {"success": 1} + with sqlite3.connect(worker.control_store) as control: + events: list[tuple[str, str]] = control.execute( + "SELECT event_type,payload FROM recall_events WHERE run_id=? " + "AND event_type IN ('run_check_started','run_check_finished') " + "ORDER BY storage_id", + (run["run_id"],), + ).fetchall() + assert [(kind, json.loads(payload)["change"]["change"]) for kind, payload in events] == [ + ("run_check_started", {"kind": "check_started", "node_key": "check:orders_valid"}), + ( + "run_check_finished", + { + "kind": "check_finished", + "node_key": "check:orders_valid", + "outcome": test_case.expected_check_outcome, + }, + ), + ] + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "actual attached check failure starts a sensor destination", + "SELECT 17 AS amount", + "SELECT amount FROM inventory", + "success", + [(59,)], + files={ + "python/checks/orders.py": "failed", + "orchestration/orders.py": "on_failure", + "models/inventory.sql": "inventory", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_attached_check_failure_when_a_sensor_observes_it_then_its_destination_executes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/attached_checks.json").read_text() + ) + test_case = replace( + test_case, files={path: fixture[key] for path, key in test_case.files.items()} + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + source: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert source["status"] == "failed", worker.log.read_text() + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="refresh_inventory" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + assert destination["selection_json"] == ["model:inventory"] + assert destination["snapshot_id"] == source["snapshot_id"] + assert source["selection_json"] == ["model:orders"] + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "cancelling active attached-check SQL cannot set failure health", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "cancelled", + [(17,)], + files={ + "python/checks/orders.py": "cancelled", + "orchestration/orders.py": "on_failure", + "models/inventory.sql": "inventory", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_failure_health( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/attached_checks.json").read_text() + ) + test_case = replace( + test_case, files={path: fixture[key] for path, key in test_case.files.items()} + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker, job_key="annual_orders") + run: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + outcomes: list[tuple[int, str]] = control.execute( + "SELECT storage_id,payload -> 'change' -> 'change' ->> 'outcome' " + "FROM recall_events WHERE run_id=? AND event_type='run_check_finished'", + (run["run_id"],), + ).fetchall() + assert [outcome for _, outcome in outcomes] == ["cancelled"] + deadline: float = time.monotonic() + 15 + cursor: int = 0 + while cursor < outcomes[0][0]: + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + cursor = control.execute( + "SELECT COALESCE(MAX(CAST(cursor_json ->> 'event_storage_id' AS INTEGER)),0) " + "FROM sqb_sensor_cursor_commits" + ).fetchone()[0] + time.sleep(0.02) + assert control.execute("SELECT count(*) FROM sqb_run_requests").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE job_key='refresh_inventory'" + ).fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ From b8beb54d43f9535a85927ad182bf535f9207dbf6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:47:50 +0000 Subject: [PATCH 108/138] fix: preserve native inputs and durable denial metadata Restore execution-input capture without relaxing coverage. Expose existing durable authorization decision IDs through the typed error body without changing permissions or error text. Keep cleanup fault injection scoped and release its sleeping backend after reset. Move blocking SQLite fault setup off the async runtime and make driver-cancellation work independent of machine speed. --- .../orchestration/attached_checks.json | 2 +- .../orchestration/warehouse_statistics.json | 2 +- clients/typescript/src/schema.d.ts | 5 + clients/typescript/tests/client.test.ts | 11 ++ .../src/stage_capture/constants.rs | 1 + .../src/api/_helpers/authentication/bearer.rs | 4 + .../authentication/password_transport.rs | 2 + .../api/_helpers/resources/docker_image.rs | 2 + .../src/api/_helpers/resources/run_reads.rs | 1 + .../src/api/_helpers/resources/runs.rs | 3 + .../src/api/_helpers/resources/snapshots.rs | 4 + .../sqlbuild-orchestration/src/api/errors.rs | 7 + .../sqlbuild-orchestration/src/api/models.rs | 3 + .../tests/api/test_contract.rs | 10 ++ .../tests/api/test_statistics_recovery.rs | 2 + .../tests/store/faults/helpers.rs | 33 +++-- .../publication/_test_types.py | 6 + .../publication/test_denial.py | 129 ++++++++++++++++++ 18 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json index d29390f156..6685c05cab 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -2,7 +2,7 @@ "passed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.pass_()\n", "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", - "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('SELECT sum(i) FROM range(10000000000) AS t(i)')\n return ctx.pass_()\n", + "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json index 640a05f0b8..3c99647743 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json @@ -11,7 +11,7 @@ "cleanup_interruption": { "block_sql": "CREATE FUNCTION pause_statistics_cleanup() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(30); RETURN NULL; END $$; CREATE TRIGGER pause_statistics_cleanup BEFORE DELETE ON sqb_warehouse_statistics FOR EACH STATEMENT EXECUTE FUNCTION pause_statistics_cleanup();", "wait_sql": "DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '5 seconds'; BEGIN WHILE NOT EXISTS (SELECT 1 FROM pg_locks l JOIN pg_stat_activity a ON a.pid=l.pid WHERE l.relation='sqb_warehouse_statistics'::regclass AND a.wait_event='PgSleep' AND a.query LIKE 'DELETE FROM sqb_warehouse_statistics%') LOOP IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'cleanup did not start'; END IF; PERFORM pg_stat_clear_snapshot(); PERFORM pg_sleep(0.02); END LOOP; END $$;", - "restore_sql": "DROP TRIGGER pause_statistics_cleanup ON sqb_warehouse_statistics; DROP FUNCTION pause_statistics_cleanup();" + "restore_sql": "SELECT pg_cancel_backend(a.pid) FROM pg_stat_activity a JOIN pg_locks l ON l.pid=a.pid WHERE l.relation='sqb_warehouse_statistics'::regclass AND a.wait_event='PgSleep' AND a.query LIKE 'DELETE FROM sqb_warehouse_statistics%'; DROP TRIGGER pause_statistics_cleanup ON sqb_warehouse_statistics; DROP FUNCTION pause_statistics_cleanup();" }, "record": { "state_database": null, diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 17d1493129..fec8beb80d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -781,6 +781,11 @@ export interface components { */ AdministratorChange: "grant" | "revoke" | "disable" | "enable"; ApiErrorBody: { + /** + * Format: uuid + * @description Existing durable denial audit ID; absent when no decision was recorded. + */ + decision_id?: string | null; error: string; }; AuditResultsBatch: { diff --git a/clients/typescript/tests/client.test.ts b/clients/typescript/tests/client.test.ts index 6bff0fd253..b600b43bd1 100644 --- a/clients/typescript/tests/client.test.ts +++ b/clients/typescript/tests/client.test.ts @@ -126,6 +126,17 @@ test("generated client preserves native scopes, replay, path encoding, SSE and b assert.equal((await browser.GET("/api/auth/session")).response.status, 401); const session = await browser.GET("/api/auth/session", { headers }); assert.deepEqual(session.data, login.data.identity); + const denied = await browser.GET("/api/control/locations/{location_id}/targets/{target}/runs", { + headers, params: { path: { location_id, target: "development" } }, + }); + assert.equal(denied.response.status, 403); + assert.equal(denied.response.headers.get("cache-control"), "no-store"); + const decisionId: string | null | undefined = denied.error?.decision_id; + assert.ok(decisionId); + assert.deepEqual(denied.error, { + error: `missing permission in this scope; all requested nodes denied; decision ${decisionId}`, + decision_id: decisionId, + }); const machine = await browser.GET(healthPath, { headers, params }); assert.equal(machine.response.status, 401); const invalidLogout = await browser.POST("/api/auth/logout", { diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index 816052a6f1..8b773cc563 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -255,6 +255,7 @@ pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 15] = [ "external_sql_reference_resolver", "scope_index", "sql_expansions", + "python_execution_inputs", ], ), ( diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs index 317bb68c88..fdf2cdb3ec 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs @@ -59,6 +59,7 @@ fn unauthorized() -> ApiError { ApiError { status: StatusCode::UNAUTHORIZED, message: "valid bearer credential required".into(), + decision_id: None, } } @@ -85,6 +86,7 @@ pub(crate) async fn local_request( return Err(ApiError { status: StatusCode::FORBIDDEN, message: "development API requires a local authority and origin".into(), + decision_id: None, }); } Ok(next.run(request).await) @@ -98,6 +100,7 @@ pub(crate) fn require_location( return Err(ApiError { status: StatusCode::FORBIDDEN, message: "credential does not authorize this location".into(), + decision_id: None, }); } Ok(()) @@ -135,6 +138,7 @@ pub(crate) async fn require_resource( return Err(ApiError { status: StatusCode::NOT_FOUND, message: missing.into(), + decision_id: None, }); } Ok(()) diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs index 9927f449cb..cd21401204 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -58,6 +58,7 @@ pub(crate) async fn guard( .ok_or_else(|| ApiError { status: StatusCode::INTERNAL_SERVER_ERROR, message: "trusted request peer is unavailable".into(), + decision_id: None, })? .0; let context = AuditContext { @@ -100,6 +101,7 @@ pub(crate) async fn authenticate( .ok_or_else(|| ApiError { status: StatusCode::INTERNAL_SERVER_ERROR, message: "trusted authentication context is unavailable".into(), + decision_id: None, })?; let access = if request.method().is_safe() { SessionAccess::Read diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs index b9d42619cd..f0e33e5df1 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs @@ -27,6 +27,7 @@ pub(crate) async fn pin( return Err(ApiError { status: StatusCode::CONFLICT, message: "published image differs from the configured Docker location image".into(), + decision_id: None, }); } input.image_id = Some(image); @@ -70,5 +71,6 @@ fn unavailable() -> ApiError { ApiError { status: StatusCode::SERVICE_UNAVAILABLE, message: "cannot inspect the configured Docker location image".into(), + decision_id: None, } } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs index 1cfc321bbd..ae2dc26f62 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs @@ -99,6 +99,7 @@ pub(crate) async fn recall( return Err(ApiError { status: StatusCode::NOT_FOUND, message: "run invocation not found".into(), + decision_id: None, }); } Ok(Json( diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs index 14b109e765..5933d5e27d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs @@ -34,6 +34,7 @@ pub(crate) async fn execution( .ok_or_else(|| ApiError { status: StatusCode::UNAUTHORIZED, message: "run claim is no longer active".into(), + decision_id: None, })?; Ok(Json(execution)) } @@ -54,6 +55,7 @@ pub(crate) async fn report( return Err(ApiError { status: StatusCode::FORBIDDEN, message: "report does not match the authenticated run and claim".into(), + decision_id: None, }); } let applied = change_run(&store, RunAuthority::Worker { run_id, claim_id }, &request).await?; @@ -120,5 +122,6 @@ fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result, } impl From for ApiError { @@ -29,6 +31,7 @@ impl From for ApiError { ControlError::Invalid(message) => Self { status: StatusCode::BAD_REQUEST, message, + decision_id: None, }, ControlError::Denied { reason, @@ -36,14 +39,17 @@ impl From for ApiError { } => Self { status: StatusCode::FORBIDDEN, message: format!("{reason}; decision {decision_id}"), + decision_id: Some(decision_id), }, ControlError::StaleGeneration | ControlError::Fenced => Self { status: StatusCode::CONFLICT, message: error.to_string(), + decision_id: None, }, _ => Self { status: StatusCode::INTERNAL_SERVER_ERROR, message: "control-store operation failed".into(), + decision_id: None, }, } } @@ -55,6 +61,7 @@ impl IntoResponse for ApiError { self.status, Json(ApiErrorBody { error: self.message, + decision_id: self.decision_id, }), ) .into_response(); diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 1f5162f8f7..39a3c0f31d 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -113,6 +113,9 @@ pub struct HeartbeatResponse { #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct ApiErrorBody { pub error: String, + /// Existing durable denial audit ID; absent when no decision was recorded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decision_id: Option, } /// SSE invalidation topics; `all` means refetch the authorized REST resources. diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 4b311bbb1b..53c0771e14 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -92,6 +92,16 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie StatusCode::UNAUTHORIZED, "authentication precedes JSON parsing" ); + assert_eq!(response.headers()["cache-control"], "no-store"); + assert_eq!(response.headers()["www-authenticate"], "Bearer"); + assert_eq!( + response + .json::() + .await + .expect("unaudited rejection"), + json!({"error": "valid bearer credential required"}), + "a response without a durable decision must not invent an ID" + ); let openapi: serde_json::Value = client .get(format!("{}/api/openapi.json", api.url)) .send() diff --git a/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs index 35d3f76efa..87b51451cb 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs @@ -51,6 +51,8 @@ fn given_statistics_cleanup_when_connection_resets_then_http_survives() { .expect("actual prune is active"); proxy.reset.send_modify(|epoch| *epoch += 1); tokio::time::sleep(Duration::from_millis(100)).await; + // A TCP reset leaves the backend sleeping with the shared database's write lock. + // Cancel only this fixture's query after injecting the reset, before dropping its trigger. sqlx_core::raw_sql::raw_sql(fault["restore_sql"].as_str().expect("restore SQL")) .execute(pool) .await diff --git a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs index 699304e9a7..ae85933743 100644 --- a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -9,13 +9,19 @@ pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault let ControlStore::Sqlite(path, _) = store else { unreachable!() }; - rusqlite::Connection::open(path) - .expect("fault connection") - .execute_batch(&format!( - "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", - point.operation, point.table, point.condition, point.sqlite_body - )) - .expect(point.description); + let path = path.clone(); + let sql = format!( + "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", + point.operation, point.table, point.condition, point.sqlite_body + ); + tokio::task::spawn_blocking(move || { + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(&sql) + }) + .await + .expect("join fault setup") + .expect(point.description); return; }; sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) @@ -30,10 +36,15 @@ pub(crate) async fn remove_fault(store: &ControlStore, point: &TransactionFault) let ControlStore::Sqlite(path, _) = store else { unreachable!() }; - rusqlite::Connection::open(path) - .expect("fault connection") - .execute_batch("DROP TRIGGER sqb_test_fault") - .expect("remove SQLite transaction fault"); + let path = path.clone(); + tokio::task::spawn_blocking(move || { + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch("DROP TRIGGER sqb_test_fault") + }) + .await + .expect("join fault removal") + .expect("remove SQLite transaction fault"); return; }; sqlx_core::query::query(&format!("DROP TRIGGER sqb_test_fault ON {}", point.table)) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b2319ab00d..0baeaa122a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -256,3 +256,9 @@ class CatalogBoundaryTestCase: class StatisticsRecoveryTestCase: description: str expected_catalog_presence: tuple[bool, ...] + + +@dataclass(frozen=True) +class DenialMetadataTestCase: + description: str + expected_denial_count: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py new file mode 100644 index 0000000000..7b45186f3e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py @@ -0,0 +1,129 @@ +"""Denial metadata identifies durable decisions without granting access to their scope.""" + +import json +import sqlite3 +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from uuid import UUID, uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DenialMetadataTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DenialMetadataTestCase( + description="durable ID without scope disclosure", expected_denial_count=6 + ) + ], + ids=lambda case: case.description, +) +def test_given_denied_browser_requests_when_returned_then_only_durable_decision_metadata_is_exposed( + test_case: DenialMetadataTestCase, + tmp_path: Path, +) -> None: + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + administrator: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-reader" + ) + admin_session: tuple[str, str] = person_login(api, administrator["name"]) + session: tuple[str, str] = person_login(api, person["name"]) + catalog: str = ( + f"/api/control/locations/{api.location_id}/targets/development/catalog?schema=main" + ) + unlisted: str = catalog.replace(api.location_id, str(uuid4())) + decisions: dict[str, str] = {} + for path in [catalog, unlisted, "/api/control/audit"]: + with pytest.raises(HTTPError) as rejected: + person_read(api, path, session=session) + assert rejected.value.code == 403 + assert rejected.value.headers["Cache-Control"] == "no-store" + body: dict[str, Any] = json.load(rejected.value) + assert set(body) == {"error", "decision_id"} + decision: str = str(UUID(body["decision_id"])) + assert body["error"] == ( + "missing permission in this scope; all requested nodes denied; decision " + decision + ) + decisions[decision] = "authorization.denied" + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": api.location_id, + "target": "development", + "job_key": "unlisted_orders_job", + "reason": "Refresh order totals", + "priority": 0, + } + for credentials, expected_status, action in [ + (session, 403, "authorization.denied"), + ((session[0], "wrong-csrf"), 401, "authentication.denied"), + (("", ""), 401, "authentication.denied"), + ]: + with pytest.raises(HTTPError) as rejected: + person_request(api, "/api/control/runs", body=request, session=credentials) + assert rejected.value.code == expected_status + assert rejected.value.headers["Cache-Control"] == "no-store" + body = json.load(rejected.value) + assert set(body) == {"error", "decision_id"} + decision = str(UUID(body["decision_id"])) + assert ( + body["error"] + == { + "authorization.denied": ( + "missing permission in this scope; all requested nodes denied; decision " + + decision + ), + "authentication.denied": "authentication rejected", + }[action] + ) + assert decision not in decisions + decisions[decision] = action + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "viewer", + "location_id": api.location_id, + "target": "development", + } + assert person_request(api, "/api/admin/roles", body=assignment, session=admin_session) == { + "applied": True + } + assert person_read(api, catalog, session=session) == {"records": [], "next_after": None} + with pytest.raises(HTTPError) as rejected: + person_read(api, unlisted, session=session) + assert rejected.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as rejected: + person_read(api, catalog, session=session) + assert rejected.value.code == 403 + events: list[dict[str, Any]] = person_read( + api, "/api/control/audit?limit=100", session=admin_session + ) + by_id: dict[str, dict[str, Any]] = {event["event_id"]: event for event in events} + assert len(decisions) == test_case.expected_denial_count + for decision, action in decisions.items(): + assert by_id[decision]["action"] == action + assert by_id[decision]["after_json"]["allowed"] is False + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as store: + assert store.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + assert store.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) From c0a352ef12e1640cde36a4a12e98885a534b384a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:27:15 +0000 Subject: [PATCH 109/138] fix: isolate native PostgreSQL contract databases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schemas do not isolate PostgreSQL advisory locks. Give unrelated native contracts separate disposable databases while retaining each contract’s schema, replicas and production writer fence. Keep default test concurrency, deadline budgets and all assertions unchanged. Update harness feedback to describe database isolation. --- .../tests/store/helpers.rs | 18 +++++++++++++++--- .../tests/store/test_types.rs | 2 +- scripts/native_tests/main/run.py | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 9893d5268e..63af06e2d7 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -40,10 +40,22 @@ pub(crate) async fn open_test_store(backend: Backend) -> Option { .await .expect("connect to test PostgreSQL"); let schema = format!("sqb_test_{}", Uuid::now_v7().simple()); - sqlx_core::query::query(&format!("CREATE SCHEMA {schema}")) + // Writer advisory locks are database-wide, so schemas alone do not isolate contracts. + sqlx_core::query::query(&format!("CREATE DATABASE {schema} TEMPLATE template0")) .execute(&admin) .await + .expect("create isolated test database"); + let mut isolated_url = reqwest::Url::parse(&url).expect("test PostgreSQL URL"); + isolated_url.set_path(&schema); + let url = isolated_url.as_str(); + let setup = sqlx_postgres::PgPool::connect(url) + .await + .expect("connect to isolated test database"); + sqlx_core::query::query(&format!("CREATE SCHEMA {schema}")) + .execute(&setup) + .await .expect("create isolated test schema"); + setup.close().await; let separator = ['?', '&'][usize::from(url.contains('?'))]; let store = open_store(&format!("{url}{separator}options=-csearch_path%3D{schema}")) .await @@ -60,10 +72,10 @@ pub(crate) async fn close_test_store(test_store: TestStore) { let Some((admin, schema)) = test_store.postgres else { return; }; - sqlx_core::query::query(&format!("DROP SCHEMA {schema} CASCADE")) + sqlx_core::query::query(&format!("DROP DATABASE {schema}")) .execute(&admin) .await - .expect("remove isolated test schema"); + .expect("remove isolated test database"); admin.close().await; drop(test_store.directory); } diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs index 1ffddfd679..d58d5beb56 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -1,4 +1,4 @@ -//! Store fixtures retain their isolated SQLite directory or PostgreSQL schema until cleanup. +//! Store fixtures retain their isolated SQLite directory or PostgreSQL database until cleanup. use crate::store::models::ControlStore; diff --git a/scripts/native_tests/main/run.py b/scripts/native_tests/main/run.py index fa3a52fc5e..e8c9ea392e 100644 --- a/scripts/native_tests/main/run.py +++ b/scripts/native_tests/main/run.py @@ -1,4 +1,4 @@ -"""One disposable database per native suite; Rust fixtures isolate their own schemas.""" +"""One disposable PostgreSQL server per suite; Rust fixtures isolate their own databases.""" from __future__ import annotations @@ -17,7 +17,7 @@ def run_native_tests(argv: list[str] | None = None) -> int: print("Starting disposable PostgreSQL for native integration tests...", flush=True) try: with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: - print("PostgreSQL ready. Running native tests with isolated schemas...", flush=True) + print("PostgreSQL ready. Running native tests with isolated databases...", flush=True) result: subprocess.CompletedProcess[bytes] = subprocess.run( ["cargo", "test", *arguments], env=os.environ From f2888c5ef907abca61da4a2f040ec323ec64a92e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:23:40 +0000 Subject: [PATCH 110/138] test: fix attached-check cancellation query binding --- .../duckshop/edge_cases/orchestration/attached_checks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json index 6685c05cab..79e6d5d12c 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -2,7 +2,7 @@ "passed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.pass_()\n", "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", - "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", + "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) AS seed(i) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" } From 77d56964f40cc334a5ea6bd56a8971ae968fedfd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:38:58 +0000 Subject: [PATCH 111/138] test: retain private delta comparison failure evidence --- .github/diagnostics-recipient.pem | 25 ++ .github/workflows/verify.yml | 15 ++ scripts/delta_compile_series/README.md | 48 ++++ .../_helpers/encryption.py | 85 +++++++ .../delta_compile_series/_helpers/evidence.py | 216 ++++++++++++++++++ scripts/delta_compile_series/constants.py | 10 + .../main/run_delta_compile_series.py | 37 ++- scripts/delta_compile_series/models.py | 2 + .../cli/commands/main/compile/helpers.py | 6 +- .../main/compile/test_compile_delta.py | 58 +++++ .../delta_compile_series/main/_test_types.py | 15 ++ .../delta_compile_series/main/helpers.py | 122 ++++++++++ .../main/test_comparison_evidence.py | 180 +++++++++++++++ .../main/test_encrypted_evidence.py | 144 ++++++++++++ 14 files changed, 961 insertions(+), 2 deletions(-) create mode 100644 .github/diagnostics-recipient.pem create mode 100644 scripts/delta_compile_series/README.md create mode 100644 scripts/delta_compile_series/_helpers/encryption.py create mode 100644 scripts/delta_compile_series/_helpers/evidence.py create mode 100644 tests/unit/scripts/delta_compile_series/main/_test_types.py create mode 100644 tests/unit/scripts/delta_compile_series/main/helpers.py create mode 100644 tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py create mode 100644 tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py diff --git a/.github/diagnostics-recipient.pem b/.github/diagnostics-recipient.pem new file mode 100644 index 0000000000..93ede90365 --- /dev/null +++ b/.github/diagnostics-recipient.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEJTCCAo2gAwIBAgIUUgxZ6C8ykh+91sFO8z78yEl0flIwDQYJKoZIhvcNAQEL +BQAwIjEgMB4GA1UEAwwXc3FsYnVpbGQtY2ktZGlhZ25vc3RpY3MwHhcNMjYxMTAz +MDQ0ODU4WhcNMzYxMDMxMDQ0ODU4WjAiMSAwHgYDVQQDDBdzcWxidWlsZC1jaS1k +aWFnbm9zdGljczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBANhDATXs +XtI0hyxJIba3/APxf8cVEZ7dTTHhyqyEpf2Pi5VKWvfLNkTJE+FIlXS37G5o4Hkb +Tnm44fWdQg/UAnQCY5J9QjEGJQAIv+4jx9jcALtbh0VG0hXHuEXBMFi3pakazhVf +rLHUtI4pDuU4Z0Zl7DsivZTgDiOAyCE+uIyCcKg5sn3vq3ITej7vIXaPHbrLV794 +M6hyPbQu8kiREW+0g6iktAiHiw4prYBlbGIVfWoCzkNUEtO6djMKQlg1es8bhCYo +cVOwChPJrgsTX4tpN7I1y6kAYQ3EAd9AcoG+YmZ/Lf0wsawH4BrM0XZWV7Tr93CO +nrLqegrCx8eYn9cweoCsWq4Ad5xDecYnbaxX3c9bFfuDmaU8tUKjm4KXTQgJFznW +0cc1dRSbO7W8ogZyBj9KvRoJPboHiXaZQSHwcx/rwKmjZJ6yDcGTlTD7qY9WciuR +KK5mWLJvsQ33fjMTGnaW5cd73UR3Y6phvHv1s4HpVeAWOjWJAVuEzDjU1QIDAQAB +o1MwUTAdBgNVHQ4EFgQUg0RhY1QY4ID4KbGS710v48wx+jQwHwYDVR0jBBgwFoAU +g0RhY1QY4ID4KbGS710v48wx+jQwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAYEAlMrRv+zYmWbdSTM5vZM1qJMlDifRbRqTqcbMzv8W0tOrm+Ch2b94 +VaPW0lCs+hOEUfAP2osegfLS6eAlv3MW774iVZzEcHjoQpvQQ2biaPgpJTe4jV+U +iwWlYrBbOjFPOQsnvxHSKjsrkt5bHcEXxShSkCYLjS+HvUT2huZ5LGxBUR9yfia2 +gkOIa1zlPlbAMyUt6Fj66NL2+nbsCYLuGfQafqF4UUd3rG2sF1pkxSEuof1VpKTQ +rKU9u0+csNFHF20CGZ7h9jUU1ATLLwBlPunFCCO70gIGcoNspqPRIOwsEXXGr81N +aXJAeA5XMesS/v0k0x2GBLXujTT5+kC3iQA0+OfxH8t82D4xTXd2wYIKYwnzK+nv +XkF3Of+Y/puUv7R+ADfG0K7aMdLkBDjllJJL7gUml+X281m+2S5UlJZIHxe2kYGB +/i2XCahytLRBY9EGVSLEWjkhfnOydpjwuHfjkNaJDqNRDexMbzifxi0/liv8jbvV +ZCqMfibxPUoT +-----END CERTIFICATE----- diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 08ae232070..cd908d1d90 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -393,8 +393,23 @@ jobs: f"{os.environ.get('PYTEST_XDIST_AUTO_NUM_WORKERS', '')}" ) PY + - name: Validate diagnostic encryption recipient + if: matrix.target == 'test-e2e-duckdb-cli' + run: openssl x509 -in .github/diagnostics-recipient.pem -noout - name: Run DuckDB E2E shard + env: + SQLBUILD_COMPARISON_ENCRYPTED_DIR: ${{ matrix.target == 'test-e2e-duckdb-cli' && format('{0}/comparison-encrypted', runner.temp) || '' }} + SQLBUILD_COMPARISON_RECIPIENT: ${{ github.workspace }}/.github/diagnostics-recipient.pem run: make ${{ matrix.target }} + - name: Retain encrypted comparison failures + if: failure() && matrix.target == 'test-e2e-duckdb-cli' + uses: actions/upload-artifact@v4 + with: + name: encrypted-comparison-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/comparison-encrypted/failure-*/failure.cms + if-no-files-found: warn + compression-level: 0 + retention-days: 7 compiler_differential: name: Compiler golden outputs diff --git a/scripts/delta_compile_series/README.md b/scripts/delta_compile_series/README.md new file mode 100644 index 0000000000..fed5784efb --- /dev/null +++ b/scripts/delta_compile_series/README.md @@ -0,0 +1,48 @@ +# Private comparison diagnostics + +CLI comparison tests keep bounded evidence only when the existing delta/full comparator +reports a mismatch. Acceptance, output normalization, progress routing and cache policy are +unchanged. Successful comparisons create no evidence. Local evidence directories use mode +0700 and files use 0600. Streams and named environment credentials are redacted, but the +remaining project content must still be treated as private. + +The CLI CI shard enables encrypted retention with `SQLBUILD_COMPARISON_ENCRYPTED_DIR` and +`SQLBUILD_COMPARISON_RECIPIENT`. The latter names the reviewed public recipient certificate; +no private key is available to the workflow. OpenSSL CMS uses AES-256-GCM and RSA-OAEP with +SHA-256 to encrypt each ZIP bundle. Only completed `failure.cms` files are uploaded, only +when the shard fails, with seven-day retention. Plaintext temporary evidence and partial +encryption output are never upload paths or printed by capture. + +Each comparison series retains at most four failures of 4 MiB each, with individual streams +capped at 512 KiB. One shard reserves at most twelve ciphertext slots, each capped at +4 MiB + 64 KiB (48.75 MiB total). Slots are reserved atomically across test workers. +Truncation, original lengths/digests and capture errors are explicit in the evidence. +Encryption failure cannot replace the original comparison result and has no plaintext +upload fallback. A missing encrypted artifact is not evidence that a failed comparison +was captured successfully. + +## Recover an encrypted failure + +Download the encrypted GitHub artifact to a private machine. Obtain the matching private +key through the private secret store, never through GitHub, repository content or logs. +Decrypt to a temporary file, and rename it only after authentication succeeds: + +```sh +umask 077 +if openssl cms -decrypt -binary -inform DER -in failure.cms \ + -inkey /private/path/recipient-key.pem -out evidence.zip.partial; then + mv evidence.zip.partial evidence.zip +else + rm -f evidence.zip.partial + exit 1 +fi +``` + +Do not inspect or extract partial decryption output: OpenSSL may write unauthenticated +bytes before rejecting tampering. Wrong keys, truncated ciphertext and modified GCM tags +must fail. Extract only into a private directory; do not publish raw evidence. + +The tracked certificate is public encryption material, not a trust root for authentication. +To rotate it, securely store and verify the replacement private key before replacing the +certificate. Retain prior private keys at least until every corresponding artifact has +expired. Neither key rotation nor diagnostics grants product access or changes CLI semantics. diff --git a/scripts/delta_compile_series/_helpers/encryption.py b/scripts/delta_compile_series/_helpers/encryption.py new file mode 100644 index 0000000000..b7abc7b414 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/encryption.py @@ -0,0 +1,85 @@ +"""Authenticated, bounded CI retention; only complete ciphertext is uploadable.""" + +from __future__ import annotations + +import io +import os +import subprocess +import zipfile +from pathlib import Path + +from scripts.delta_compile_series.constants import ( + MAX_ENCRYPTED_BYTES, + MAX_ENCRYPTED_FAILURES, + MAX_EVIDENCE_FILES, + MAX_STEP_BYTES, +) + + +def encrypt_comparison_failure(directory: Path) -> None: + """Retain at most twelve encrypted failures when CI explicitly enables retention.""" + + configured: str | None = os.environ.get("SQLBUILD_COMPARISON_ENCRYPTED_DIR") + if not configured: + return + destination: Path = Path(configured) + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + destination.chmod(0o700) + for index in range(MAX_ENCRYPTED_FAILURES): + slot: Path = destination / f"failure-{index:02d}" + try: + slot.mkdir(mode=0o700) + except FileExistsError: + continue + break + else: + return + pending: Path = slot / "encrypted.partial" + try: + files: list[Path] = list(directory.iterdir()) + if len(files) > MAX_EVIDENCE_FILES or any( + path.is_symlink() or not path.is_file() for path in files + ): + raise OSError("Invalid comparison evidence files") + archive: io.BytesIO = io.BytesIO() + remaining: int = MAX_STEP_BYTES + with zipfile.ZipFile(archive, mode="w", compression=zipfile.ZIP_DEFLATED) as bundle: + for path in sorted(files): + with path.open("rb") as stream: + data: bytes = stream.read(remaining + 1) + remaining -= len(data) + if remaining < 0: + raise OSError("Comparison evidence exceeds retention bound") + bundle.writestr(path.name, data) + certificate: str = os.environ.get("SQLBUILD_COMPARISON_RECIPIENT", "") + with pending.open("xb") as output: + pending.chmod(0o600) + result: subprocess.CompletedProcess[bytes] = subprocess.run( + [ + "openssl", + "cms", + "-encrypt", + "-binary", + "-aes-256-gcm", + "-outform", + "DER", + "-recip", + certificate, + "-keyopt", + "rsa_padding_mode:oaep", + "-keyopt", + "rsa_oaep_md:sha256", + ], + input=archive.getvalue(), + stdout=output, + stderr=subprocess.PIPE, + check=False, + timeout=30, + ) + if result.returncode != 0 or not 0 < pending.stat().st_size <= MAX_ENCRYPTED_BYTES: + raise OSError("Comparison evidence encryption failed") + pending.replace(slot / "failure.cms") + except subprocess.TimeoutExpired as error: + raise OSError("Comparison evidence encryption timed out") from error + finally: + pending.unlink(missing_ok=True) diff --git a/scripts/delta_compile_series/_helpers/evidence.py b/scripts/delta_compile_series/_helpers/evidence.py new file mode 100644 index 0000000000..852f68ae53 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/evidence.py @@ -0,0 +1,216 @@ +"""Bounded, private evidence for a failed delta/full comparison, never CLI output.""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from itertools import islice +from pathlib import Path +from typing import Any, cast + +from scripts.delta_compile_series._helpers.encryption import encrypt_comparison_failure +from scripts.delta_compile_series._helpers.outcomes import _seen +from scripts.delta_compile_series.constants import ( + COMPARISON_MANIFEST_FILE, + MAX_CACHE_PATHS, + MAX_ENTRIES, + MAX_FAILURES, + MAX_FILE_BYTES, + MAX_READ_BYTES, + MAX_STEP_BYTES, +) +from scripts.delta_compile_series.models import DeltaCompileOutcome +from sqlbuild._native import redact_diagnostic_text, redact_diagnostic_value + +_MANIFEST_RESERVE: int = 64 * 1024 + + +def _file_state(*, path: Path, root: Path, limit: int = MAX_FILE_BYTES) -> dict[str, Any]: + """Stat and bounded content digest, without following paths outside the project.""" + + if not path.resolve().is_relative_to(root.resolve()): + return {"outside_project": True} + if not path.is_file(): + return {"missing": True} + status: os.stat_result = path.stat() + with path.open("rb") as stream: + data: bytes = stream.read(limit) + return { + "size": status.st_size, + "mtime_ns": status.st_mtime_ns, + "ctime_ns": status.st_ctime_ns, + "sha256": hashlib.sha256(data).hexdigest(), + "hashed_bytes": len(data), + "digest_truncated": len(data) != status.st_size, + } + + +def cache_state(project: Path) -> dict[str, Any]: + """Report evidence-collection errors without changing the comparison or CLI outcome.""" + + try: + return _cache_state(project) + except (OSError, ValueError) as error: + return {"capture_error": str(error)} + + +def _cache_state(project: Path) -> dict[str, Any]: + """Snapshot bounded SQL-test records and physical reuse evidence, not inferred identities.""" + + snapshots: dict[str, Any] = {} + remaining: int = MAX_READ_BYTES + paths: list[Path] = list( + islice((project / "target/cache").rglob("sql-test-artifacts.json"), MAX_CACHE_PATHS + 1) + ) + for path in sorted(paths[:MAX_CACHE_PATHS]): + snapshot: dict[str, Any] = _file_state(path=path, root=project) + snapshots[str(path.relative_to(project))] = snapshot + if snapshot.get("outside_project") or snapshot.get("missing"): + continue + with path.open("rb") as stream: + data: bytes = stream.read(MAX_FILE_BYTES + 1) + snapshot["cache_text"] = data[:MAX_FILE_BYTES].decode("utf-8", errors="replace") + try: + payload: Any = json.loads(data) if len(data) <= MAX_FILE_BYTES else None + except (ValueError, UnicodeError): + payload = None + snapshot["parseable"] = isinstance(payload, dict) + records: Any = payload.get("records") if isinstance(payload, dict) else None + if not isinstance(records, dict): + continue + snapshot["records_truncated"] = len(records) > MAX_ENTRIES + physical: dict[str, Any] = {} + for key, raw in islice(cast(dict[str, Any], records).items(), MAX_ENTRIES): + if not isinstance(raw, dict): + continue + record: dict[str, Any] = cast(dict[str, Any], raw) + relative_path: Any = record.get("relative_path") + if not isinstance(relative_path, str): + continue + state: dict[str, Any] = _file_state( + path=project / "target/compiled/tests" / relative_path, + root=project, + limit=min(MAX_FILE_BYTES, remaining), + ) + remaining -= state.get("hashed_bytes", 0) + state["recorded_identity"] = record.get("identity") + state["recorded_size_matches"] = state.get("size") == record.get("size") + state["recorded_mtime_matches"] = state.get("mtime_ns") == record.get("mtime_ns") + state["recorded_digest_matches"] = not state.get( + "digest_truncated", True + ) and state.get("sha256") == record.get("content_sha256") + physical[key] = state + snapshot["physical_artifacts"] = physical + return { + "captured_at_ns": time.time_ns(), + "caches": snapshots, + "cache_paths_truncated": len(paths) > MAX_CACHE_PATHS, + } + + +def _artifacts(project: Path) -> dict[str, Any]: + """Bounded per-file digests of actual compiled SQL, without copying query text.""" + + paths: list[Path] = list(islice((project / "target/compiled").rglob("*.sql"), MAX_ENTRIES + 1)) + files: dict[str, Any] = {} + remaining: int = MAX_READ_BYTES + for path in sorted(paths[:MAX_ENTRIES]): + state: dict[str, Any] = _file_state( + path=path, root=project, limit=min(MAX_FILE_BYTES, remaining) + ) + remaining -= state.get("hashed_bytes", 0) + files[str(path.relative_to(project))] = state + return { + "files": files, + "truncated": len(paths) > MAX_ENTRIES, + } + + +def _saved(*, destination: Path, name: str, text: str) -> dict[str, Any]: + """Redact named credentials, preserve stream order, and explicitly mark any byte limit.""" + + original: bytes = text.encode("utf-8", errors="surrogateescape") + data: bytes = redact_diagnostic_text(text).encode("utf-8", errors="surrogateescape") + used: int = sum(path.stat().st_size for path in destination.iterdir()) + limit: int = ( + min(_MANIFEST_RESERVE, max(0, MAX_STEP_BYTES - used)) + if name == COMPARISON_MANIFEST_FILE + else min(MAX_FILE_BYTES, max(0, MAX_STEP_BYTES - _MANIFEST_RESERVE - used)) + ) + kept: bytes = data[:limit] + with (destination / name).open("xb") as stream: + os.chmod(stream.name, 0o600) + _ = stream.write(kept) + return { + "original_sha256": hashlib.sha256(original).hexdigest(), + "original_bytes": len(original), + "redacted": data != original, + "retained_bytes": len(kept), + "truncated": len(kept) != len(data), + } + + +def keep_comparison_failure( + *, + destination: Path, + seed: int, + step: str, + case: str, + project: Path, + copy: Path, + outcome: DeltaCompileOutcome, + full: DeltaCompileOutcome, + before: dict[str, Any], + after: dict[str, Any], +) -> None: + """Keep at most four 4-MiB private failures; nothing is printed or uploaded.""" + + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + destination.chmod(0o700) + failures: list[Path] = list(destination.iterdir()) + if len(failures) >= MAX_FAILURES: + return + directory: Path = destination / f"failure-{len(failures):02d}" + directory.mkdir(mode=0o700) + manifest: dict[str, Any] = {"seed": seed, "step": step, "case": case, "files": {}, "runs": {}} + for label, result, root in (("delta", outcome, project), ("full", full, copy)): + normalized: tuple[str, ...] = _seen(outcome=result, project_dir=project, other_dir=root) + try: + report: Any = json.loads(normalized[1]) + except ValueError: + report = None + manifest["runs"][label] = { + "returncode": result.returncode, + "command": result.command, + "environment": [ + (name, redact_diagnostic_value(name=name, value=value)) + for name, value in result.environment + ], + "project": str(root), + "semantic_json_sha256": hashlib.sha256( + json.dumps(report, sort_keys=True).encode() + ).hexdigest() + if report is not None + else None, + } + for stream in ("stdout", "stderr"): + name = f"{label}.{stream}" + manifest["files"][name] = _saved( + destination=directory, name=name, text=getattr(result, stream) + ) + name = f"{label}.artifacts.json" + manifest["files"][name] = _saved( + destination=directory, name=name, text=json.dumps(_artifacts(root), sort_keys=True) + ) + for name, state in (("cache-before.json", before), ("cache-after.json", after)): + manifest["files"][name] = _saved( + destination=directory, name=name, text=json.dumps(state, sort_keys=True) + ) + _ = _saved( + destination=directory, + name=COMPARISON_MANIFEST_FILE, + text=json.dumps(manifest, indent=2, sort_keys=True), + ) + encrypt_comparison_failure(directory) diff --git a/scripts/delta_compile_series/constants.py b/scripts/delta_compile_series/constants.py index edafd2c76e..3e276ef56b 100644 --- a/scripts/delta_compile_series/constants.py +++ b/scripts/delta_compile_series/constants.py @@ -24,6 +24,16 @@ FULL_COPY_SUFFIX: str = "_full_copy" DURATION_PATTERN: re.Pattern[str] = re.compile(r"\s*\(\d+(?:\.\d+)?s\)") COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}') +MAX_FAILURES: int = 4 +MAX_STEP_BYTES: int = 4 * 1024 * 1024 +MAX_FILE_BYTES: int = 512 * 1024 +MAX_ENTRIES: int = 2048 +MAX_READ_BYTES: int = 16 * 1024 * 1024 +MAX_CACHE_PATHS: int = 16 +COMPARISON_MANIFEST_FILE: str = "manifest.json" +MAX_EVIDENCE_FILES: int = 9 +MAX_ENCRYPTED_FAILURES: int = 12 +MAX_ENCRYPTED_BYTES: int = MAX_STEP_BYTES + 64 * 1024 EDITS: tuple[DeltaCompileEdit, ...] = ( DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), diff --git a/scripts/delta_compile_series/main/run_delta_compile_series.py b/scripts/delta_compile_series/main/run_delta_compile_series.py index ac38d9beba..dd8e31d65d 100644 --- a/scripts/delta_compile_series/main/run_delta_compile_series.py +++ b/scripts/delta_compile_series/main/run_delta_compile_series.py @@ -6,8 +6,10 @@ import shutil from collections.abc import Callable from pathlib import Path +from typing import Any from scripts.delta_compile_series._helpers.edits import edited_file +from scripts.delta_compile_series._helpers.evidence import cache_state, keep_comparison_failure from scripts.delta_compile_series._helpers.outcomes import full_compile_difference, judged from scripts.delta_compile_series.constants import ( DELTA_COMPILE_LINE, @@ -29,16 +31,43 @@ def _compared( project_dir: Path, compile_project: Callable[[], DeltaCompileOutcome], compile_in_full: Callable[[Path], DeltaCompileOutcome], + diagnostics_dir: Path | None, + seed: int, + step: str, + case: str, ) -> tuple[DeltaCompileOutcome, str | None]: """Compile, and compile a copy of the project as it was in full; how what they print differs.""" copy: Path = project_dir.with_name(project_dir.name + FULL_COPY_SUFFIX) shutil.rmtree(copy, ignore_errors=True) _ = shutil.copytree(project_dir, copy, symlinks=True) + before: dict[str, Any] = ( + {"delta": cache_state(project_dir), "full": cache_state(copy)} if diagnostics_dir else {} + ) outcome: DeltaCompileOutcome = compile_project() + after: dict[str, Any] = {"delta": cache_state(project_dir)} if diagnostics_dir else {} + full: DeltaCompileOutcome = compile_in_full(copy) + if diagnostics_dir: + after["full"] = cache_state(copy) problem: str | None = full_compile_difference( - outcome=outcome, full=compile_in_full(copy), project_dir=project_dir, full_project_dir=copy + outcome=outcome, full=full, project_dir=project_dir, full_project_dir=copy ) + if problem is not None and diagnostics_dir is not None: + try: + _ = keep_comparison_failure( + destination=diagnostics_dir, + seed=seed, + step=step, + case=case, + project=project_dir, + copy=copy, + outcome=outcome, + full=full, + before=before, + after=after, + ) + except OSError: + pass return outcome, problem @@ -49,6 +78,8 @@ def run_delta_compile_series( steps: int, compile_project: Callable[[], DeltaCompileOutcome], compile_in_full: Callable[[Path], DeltaCompileOutcome] | None = None, + diagnostics_dir: Path | None = None, + case: str = "", ) -> DeltaCompileSeries: """Edit or revert `steps` times, compiling after each, and judge or compare every compile.""" @@ -87,6 +118,10 @@ def run_delta_compile_series( project_dir=project_dir, compile_project=compile_project, compile_in_full=compile_in_full, + diagnostics_dir=diagnostics_dir, + seed=seed, + step=step, + case=case, ) delta: bool = DELTA_COMPILE_LINE in outcome.stderr deltas += int(delta) diff --git a/scripts/delta_compile_series/models.py b/scripts/delta_compile_series/models.py index 26628e2dfc..367bce381d 100644 --- a/scripts/delta_compile_series/models.py +++ b/scripts/delta_compile_series/models.py @@ -25,6 +25,8 @@ class DeltaCompileOutcome: returncode: int stdout: str stderr: str + command: tuple[str, ...] = () + environment: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index caaf69d744..801b9dc17d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5318,7 +5318,11 @@ def run_delta_compile( project_dir=project_dir, args=args, env={REUSE_DISABLE_ENV_VAR: "0", **env} ) return DeltaCompileOutcome( - returncode=result.returncode, stdout=result.stdout, stderr=result.stderr + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + command=tuple(result.args), + environment=tuple(sorted({REUSE_DISABLE_ENV_VAR: "0", **env}.items())), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 5786204a87..87b79f8819 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -2,9 +2,11 @@ from __future__ import annotations +import json import shutil from functools import partial from pathlib import Path +from typing import Any import pytest @@ -262,6 +264,60 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_ful assert series.deltas >= test_case.expected_minimum_deltas, series.steps +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSeriesTestCase( + description="cache mode diagnostic control", + seed=16, + steps=1, + json_output=True, + adapter=_UNTYPED_ADAPTER, + expected_minimum_deltas=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_cache_mode_mismatch_when_comparing_then_keeps_actual_cli_evidence( + tmp_path: Path, + test_case: DeltaCompileSeriesTestCase, +) -> None: + project_dir: Path = tmp_path / "orders" + destination: Path = tmp_path / "private-evidence" + write_delta_compile_project(project_dir=project_dir, model_count=30, adapter=test_case.adapter) + + series: DeltaCompileSeries = run_delta_compile_series( + project_dir=project_dir, + seed=test_case.seed, + steps=test_case.steps, + compile_project=partial( + run_delta_compile, project_dir=project_dir, args=_JSON_COMPILE, env={} + ), + compile_in_full=partial(run_full_compile, args=(*_JSON_COMPILE, "--no-cache")), + diagnostics_dir=destination, + case=test_case.description, + ) + + assert series.deltas >= test_case.expected_minimum_deltas + assert len(series.unexpected) == 1 + assert "stderr differs from a full compile" in series.unexpected[0] + directory: Path = destination / "failure-00" + manifest: dict[str, Any] = json.loads((directory / "manifest.json").read_text()) + assert manifest["runs"]["delta"]["returncode"] == manifest["runs"]["full"]["returncode"] == 0 + assert ( + manifest["runs"]["delta"]["semantic_json_sha256"] + == manifest["runs"]["full"]["semantic_json_sha256"] + ) + assert manifest["runs"]["full"]["command"][-3:] == ["compile", "--json", "--no-cache"] + assert "SQL test planning START" in (directory / "full.stderr").read_text() + assert "SQL test planning START" not in (directory / "delta.stderr").read_text() + delta: dict[str, Any] = json.loads((directory / "delta.artifacts.json").read_text())["files"] + full: dict[str, Any] = json.loads((directory / "full.artifacts.json").read_text())["files"] + assert {path: state["sha256"] for path, state in delta.items()} == { + path: state["sha256"] for path, state in full.items() + } + + @pytest.mark.parametrize( "test_case", [ @@ -291,6 +347,8 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_output_matches_f steps=test_case.steps, compile_project=partial(run_delta_compile, project_dir=project_dir, args=args, env={}), compile_in_full=partial(run_full_compile, args=args), + diagnostics_dir=tmp_path / "private-comparison-evidence", + case=test_case.description, ) assert series.unexpected == test_case.expected_unexpected_outcomes, series.steps diff --git a/tests/unit/scripts/delta_compile_series/main/_test_types.py b/tests/unit/scripts/delta_compile_series/main/_test_types.py new file mode 100644 index 0000000000..30c764273b --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/_test_types.py @@ -0,0 +1,15 @@ +"""Comparison evidence controls.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ComparisonEvidenceTestCase: + description: str + stderr: str = "SQL test planning START\npassword=synthetic-secret\n" + stdout: str = '{"models": ["orders"]}\n' + steps: int = 1 + expected_mismatches: int = 1 + cache_relative_path: str = "orders/check.sql" diff --git a/tests/unit/scripts/delta_compile_series/main/helpers.py b/tests/unit/scripts/delta_compile_series/main/helpers.py new file mode 100644 index 0000000000..22d1a9df03 --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/helpers.py @@ -0,0 +1,122 @@ +"""Synthetic compile callbacks exercising the comparison harness's public interface.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +from scripts.delta_compile_series.main.run_delta_compile_series import run_delta_compile_series +from scripts.delta_compile_series.main.write_delta_compile_project import ( + write_delta_compile_project, +) +from scripts.delta_compile_series.models import DeltaCompileOutcome, DeltaCompileSeries +from tests.unit.scripts.delta_compile_series.main._test_types import ComparisonEvidenceTestCase + + +def run_comparison( + *, + root: Path, + destination: Path | None, + test_case: ComparisonEvidenceTestCase, +) -> DeltaCompileSeries: + """Compare real copied cache/artifacts, with controlled output and cache changes.""" + + project: Path = root / "orders" + write_delta_compile_project(project_dir=project, model_count=8) + artifact: Path = project / "target/compiled/tests/orders/check.sql" + artifact.parent.mkdir(parents=True) + artifact.write_text("SELECT 1 AS order_id\n") + cache: Path = project / "target/cache/compiler-native-v1/sql-test-artifacts.json" + cache.parent.mkdir(parents=True) + cache.write_text( + json.dumps( + { + "version": 5, + "records": { + "orders": { + "relative_path": test_case.cache_relative_path, + "identity": "original-identity", + "size": artifact.stat().st_size, + "mtime_ns": artifact.stat().st_mtime_ns, + "content_sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), + } + }, + } + ) + ) + + def compile_project() -> DeltaCompileOutcome: + return DeltaCompileOutcome(0, test_case.stdout, "", ("sqb", "compile", "--json")) + + def compile_full(copy: Path) -> DeltaCompileOutcome: + _ = (copy / cache.relative_to(project)).write_text("{") + return DeltaCompileOutcome( + 0, + test_case.stdout, + test_case.stderr, + ("sqb", "compile", "--json"), + (("SQLBUILD_DELTA_COMPILE_DISABLE", "1"), ("PASSWORD", "synthetic-environment-secret")), + ) + + return run_delta_compile_series( + project_dir=project, + seed=16, + steps=test_case.steps, + compile_project=compile_project, + compile_in_full=compile_full, + diagnostics_dir=destination, + case="orders comparison", + ) + + +def diagnostic_recipient(root: Path) -> tuple[Path, Path]: + """Generate disposable test keys without printing or persisting them outside tmp_path.""" + + root.mkdir(mode=0o700) + key: Path = root / "key.pem" + certificate: Path = root / "recipient.pem" + _ = subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(key), + "-out", + str(certificate), + "-days", + "1", + "-subj", + "/CN=orders-diagnostics-test", + ], + check=True, + capture_output=True, + ) + key.chmod(0o600) + return certificate, key + + +def decrypt_evidence(path: Path, key: Path) -> subprocess.CompletedProcess[bytes]: + """Keep all unauthenticated output in memory; callers must check success before use.""" + + return subprocess.run( + [ + "openssl", + "cms", + "-decrypt", + "-binary", + "-inform", + "DER", + "-in", + str(path), + "-inkey", + str(key), + ], + check=False, + capture_output=True, + ) diff --git a/tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py b/tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py new file mode 100644 index 0000000000..98c2548a32 --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py @@ -0,0 +1,180 @@ +"""Private failure evidence must not change comparison acceptance or successful runs.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from scripts.delta_compile_series.models import DeltaCompileSeries +from tests.unit.scripts.delta_compile_series.main._test_types import ComparisonEvidenceTestCase +from tests.unit.scripts.delta_compile_series.main.helpers import run_comparison + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="private paired evidence")], + ids=lambda case: case.description, +) +def test_given_mismatch_when_capturing_then_keeps_private_paired_evidence( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=destination, test_case=test_case + ) + + assert len(result.unexpected) == test_case.expected_mismatches + assert ( + "stderr differs from a full compile: '' vs 'SQL test planning START'" + in result.unexpected[0] + ) + directory: Path = destination / "failure-00" + manifest: dict[str, Any] = json.loads((directory / "manifest.json").read_text()) + before: dict[str, Any] = json.loads((directory / "cache-before.json").read_text()) + after: dict[str, Any] = json.loads((directory / "cache-after.json").read_text()) + key = "target/cache/compiler-native-v1/sql-test-artifacts.json" + assert manifest["seed"] == 16 + assert manifest["case"] == "orders comparison" + assert manifest["step"].startswith("step 0: column_description") + assert manifest["runs"]["full"]["command"] == ["sqb", "compile", "--json"] + assert manifest["runs"]["full"]["environment"] == [ + ["SQLBUILD_DELTA_COMPILE_DISABLE", "1"], + ["PASSWORD", "[REDACTED]"], + ] + assert "synthetic-environment-secret" not in (directory / "manifest.json").read_text() + assert manifest["runs"]["delta"]["returncode"] == manifest["runs"]["full"]["returncode"] == 0 + assert ( + manifest["runs"]["delta"]["semantic_json_sha256"] + == manifest["runs"]["full"]["semantic_json_sha256"] + ) + assert before["full"]["caches"][key]["parseable"] is True + assert after["full"]["caches"][key]["parseable"] is False + physical: dict[str, Any] = before["delta"]["caches"][key]["physical_artifacts"]["orders"] + assert physical["recorded_digest_matches"] is True + assert physical["recorded_mtime_matches"] is True + assert physical["recorded_identity"] == "original-identity" + assert (directory / "delta.stdout").read_text() == '{"models": ["orders"]}\n' + assert "synthetic-secret" not in (directory / "full.stderr").read_text() + assert manifest["files"]["full.stderr"]["redacted"] is True + artifacts: dict[str, Any] = json.loads((directory / "delta.artifacts.json").read_text()) + assert ( + artifacts["files"]["target/compiled/tests/orders/check.sql"]["sha256"] + == hashlib.sha256(b"SELECT 1 AS order_id\n").hexdigest() + ) + assert destination.stat().st_mode & 0o777 == 0o700 + assert directory.stat().st_mode & 0o777 == 0o700 + assert all(path.stat().st_mode & 0o777 == 0o600 for path in directory.iterdir()) + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="no capture on success", stderr="", expected_mismatches=0 + ) + ], + ids=lambda case: case.description, +) +def test_given_success_when_capturing_then_changes_neither_result_nor_files( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + captured: DeltaCompileSeries = run_comparison( + root=tmp_path / "captured", destination=destination, test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + + assert captured == normal + assert len(captured.unexpected) == test_case.expected_mismatches + assert captured.unexpected == () + assert not destination.exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="bounded failures", + stdout="x" * (512 * 1024 + 19), + steps=6, + expected_mismatches=6, + ) + ], + ids=lambda case: case.description, +) +def test_given_large_repeated_mismatches_when_capturing_then_bounds_size( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=destination, test_case=test_case + ) + + assert len(result.unexpected) == test_case.expected_mismatches + assert len(list(destination.iterdir())) == 4 + assert ( + sum(path.stat().st_size for path in filter(Path.is_file, destination.rglob("*"))) + <= 16 * 1024 * 1024 + ) + manifest: dict[str, Any] = json.loads((destination / "failure-00/manifest.json").read_text()) + assert manifest["files"]["delta.stdout"]["truncated"] is True + assert ( + manifest["files"]["delta.stdout"]["original_sha256"] + == hashlib.sha256(test_case.stdout.encode()).hexdigest() + ) + assert (destination / "failure-00/delta.stdout").stat().st_size == 512 * 1024 + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="capture cannot mask mismatch")], + ids=lambda case: case.description, +) +def test_given_capture_io_failure_when_comparing_then_preserves_original_failure( + tmp_path: Path, + test_case: ComparisonEvidenceTestCase, +) -> None: + destination: Path = tmp_path / "not-a-directory" + destination.write_text("occupied") + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=destination, test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + + assert result == normal + assert len(result.unexpected) == test_case.expected_mismatches + assert destination.read_text() == "occupied" + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="malformed cache path", cache_relative_path="bad\x00.sql" + ) + ], + ids=lambda case: case.description, +) +def test_given_invalid_cache_path_when_capturing_then_preserves_comparison( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "captured", destination=destination, test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + + assert result == normal + assert len(result.unexpected) == test_case.expected_mismatches + before: dict[str, Any] = json.loads((destination / "failure-00/cache-before.json").read_text()) + assert "capture_error" in before["delta"] diff --git a/tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py b/tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py new file mode 100644 index 0000000000..9df10e2312 --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py @@ -0,0 +1,144 @@ +"""Real OpenSSL retention through the unchanged comparison interface.""" + +from __future__ import annotations + +import io +import subprocess +import zipfile +from concurrent.futures import Future, ThreadPoolExecutor +from pathlib import Path + +import pytest + +from scripts.delta_compile_series.models import DeltaCompileSeries +from tests.unit.scripts.delta_compile_series.main._test_types import ComparisonEvidenceTestCase +from tests.unit.scripts.delta_compile_series.main.helpers import ( + decrypt_evidence, + diagnostic_recipient, + run_comparison, +) + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="authenticated failure retention")], + ids=lambda case: case.description, +) +def test_given_mismatch_when_encrypting_then_decrypts_exact_evidence_and_rejects_damage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + certificate, key = diagnostic_recipient(tmp_path / "recipient") + _, wrong_key = diagnostic_recipient(tmp_path / "other-recipient") + encrypted: Path = tmp_path / "encrypted" + private: Path = tmp_path / "private" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + monkeypatch.setenv("SQLBUILD_COMPARISON_RECIPIENT", str(certificate)) + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=private, test_case=test_case + ) + assert len(result.unexpected) == test_case.expected_mismatches + ciphertext: Path = encrypted / "failure-00/failure.cms" + decrypted: subprocess.CompletedProcess[bytes] = decrypt_evidence(ciphertext, key) + assert decrypted.returncode == 0 + with zipfile.ZipFile(io.BytesIO(decrypted.stdout)) as archive: + assert set(archive.namelist()) == {path.name for path in (private / "failure-00").iterdir()} + assert all( + archive.read(name) == (private / "failure-00" / name).read_bytes() + for name in archive.namelist() + ) + assert decrypt_evidence(ciphertext, wrong_key).returncode != 0 + original: bytes = ciphertext.read_bytes() + damaged: Path = tmp_path / "damaged.cms" + damaged.write_bytes(original[:-1] + bytes([original[-1] ^ 1])) + assert decrypt_evidence(damaged, key).returncode != 0 + damaged.write_bytes(original[: len(original) // 2]) + assert decrypt_evidence(damaged, key).returncode != 0 + assert b"SQL test planning" not in original + assert ciphertext.stat().st_mode & 0o777 == 0o600 + assert not list(encrypted.rglob("*.partial")) + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="successful comparison", stderr="", expected_mismatches=0 + ) + ], + ids=lambda case: case.description, +) +def test_given_success_when_retention_enabled_then_creates_no_private_or_encrypted_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + encrypted: Path = tmp_path / "encrypted" + private: Path = tmp_path / "private" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=private, test_case=test_case + ) + assert result.unexpected == () + assert len(result.unexpected) == test_case.expected_mismatches + assert not encrypted.exists() + assert not private.exists() + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="failed encryption")], + ids=lambda case: case.description, +) +def test_given_invalid_recipient_when_retaining_then_preserves_failure_without_uploadable_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + encrypted: Path = tmp_path / "encrypted" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + monkeypatch.setenv("SQLBUILD_COMPARISON_RECIPIENT", str(tmp_path / "missing.pem")) + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "captured", destination=tmp_path / "private", test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + assert result == normal + assert len(result.unexpected) == test_case.expected_mismatches + assert not list(encrypted.rglob("*.cms")) + assert not list(encrypted.rglob("*.partial")) + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="bounded encrypted retention", + steps=4, + expected_mismatches=4, + stdout="x" * (512 * 1024 + 19), + ) + ], + ids=lambda case: case.description, +) +def test_given_repeated_failures_when_retaining_then_bounds_ciphertext_count_and_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + certificate, _ = diagnostic_recipient(tmp_path / "recipient") + encrypted: Path = tmp_path / "encrypted" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + monkeypatch.setenv("SQLBUILD_COMPARISON_RECIPIENT", str(certificate)) + with ThreadPoolExecutor(max_workers=4) as executor: + futures: list[Future[DeltaCompileSeries]] = [ + executor.submit( + run_comparison, + root=tmp_path / f"run-{index}", + destination=tmp_path / f"private-{index}", + test_case=test_case, + ) + for index in range(4) + ] + assert all( + len(future.result().unexpected) == test_case.expected_mismatches for future in futures + ) + files: list[Path] = list(encrypted.rglob("*.cms")) + assert len(files) == 12 + assert all(0 < path.stat().st_size <= 4 * 1024 * 1024 + 64 * 1024 for path in files) + assert sum(path.stat().st_size for path in files) <= 12 * (4 * 1024 * 1024 + 64 * 1024) + assert not list(encrypted.rglob("*.partial")) From 9fb35907616cceaa884c0cce7c149949ebe583e0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:29:22 +0000 Subject: [PATCH 112/138] feat: expose scoped person connection tests --- .../administrator_lifecycle.json | 7 + clients/typescript/src/schema.d.ts | 128 ++++++++++++ .../src/native_commands/main/_run_admin.rs | 2 +- .../authentication/password_sessions.rs | 8 + .../src/api/_helpers/resources/mod.rs | 1 + .../resources/person_location_work.rs | 67 +++++++ .../sqlbuild-orchestration/src/api/models.rs | 6 +- .../src/platform/_helpers/connection_tests.rs | 31 +++ .../src/platform/_helpers/mod.rs | 1 + .../src/platform/main/mod.rs | 1 + .../main/read_person_connection_test.rs | 32 +++ .../platform/main/request_location_work.rs | 14 +- .../src/platform/models.rs | 7 + .../tests/api/test_contracts.rs | 2 + .../tests/api/test_location_work.rs | 14 +- .../tests/api/test_person_location_work.rs | 183 ++++++++++++++++++ .../publication/person/helpers.py | 27 +++ .../compiled_project/publication/test_work.py | 178 +++++++++++++++++ 18 files changed, 695 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json index f66bd786c5..6a75cf5653 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -5,6 +5,13 @@ "recovery_reason": "restore access after loss of the final administrator", "disabled_person_sequence": ["assign", "disable", "revoke", "enable", "deny_run"], "run_control_sequence": ["request", "cancel", "confirm_stopped", "rerun_frozen", "retry", "revoke", "deny_retry"], + "connection_test_roles": [ + {"role": null, "allowed": false}, + {"role": "viewer", "allowed": false}, + {"role": "operator", "allowed": false}, + {"role": "admin", "allowed": true}, + {"role": "developer", "allowed": true} + ], "job_settings": [ {"full_refresh": false}, {"full_refresh": true}, diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index fec8beb80d..5d3f3d8bc6 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -179,6 +179,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/connection-tests": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestConnectionTest"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readConnectionTest"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/fingerprint": { parameters: { query?: never; @@ -847,6 +879,11 @@ export interface components { rejected_sha256: string; table: components["schemas"]["Table"]; }; + ConnectionTestRequest: { + reason: string; + /** Format: uuid */ + request_id: string; + }; /** @description The current successful snapshot, read with a consistent generation and change set. */ CurrentSnapshot: { changes: components["schemas"]["SnapshotChange"][]; @@ -2369,6 +2406,97 @@ export interface operations { }; }; }; + requestConnectionTest: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ConnectionTestRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readConnectionTest: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; readPersonFingerprint: { parameters: { query: { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 0aee19f181..f4a156f9bb 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -79,7 +79,7 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { - let applied = request_location_work(&store, &context, &request).await?; + let applied = request_location_work(&store, (&context, None), &request).await?; Ok(json!({"request_id": request.request_id, "applied": applied})) } ControlAdmin::LocationWork(request_id) => { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index c758b4443d..2f11dced21 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -58,6 +58,14 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/control/triggers", post(crate::api::_helpers::resources::person_operations::change_trigger), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/connection-tests", + post(crate::api::_helpers::resources::person_location_work::request), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", + get(crate::api::_helpers::resources::person_location_work::result), + ) .route( "/api/control/locations/{location_id}/targets/{target}/runs", get(crate::api::_helpers::resources::person_operations::recent_runs), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index cbb0575196..c85a143786 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod client; pub(crate) mod docker_image; pub(crate) mod location_health; pub(crate) mod location_work; +pub(crate) mod person_location_work; pub(crate) mod person_operations; pub(crate) mod router; pub(crate) mod run_reads; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs new file mode 100644 index 0000000000..a18cb71386 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs @@ -0,0 +1,67 @@ +//! Person connection tests reuse the location's named outbound work, not arbitrary commands. + +use axum::extract::{Path, State}; +use axum::http::HeaderMap; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::password_transport::session_credentials; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::AuditContext; +use crate::platform::main::read_person_connection_test::read_person_connection_test; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ + ConnectionTestRequest, LocationWork, LocationWorkRecord, LocationWorkRequest, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/control/locations/{location_id}/targets/{target}/connection-tests", + operation_id = "requestConnectionTest", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("x-sqb-csrf" = String, Header)), + request_body = ConnectionTestRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = [])))] +pub(crate) async fn request( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_location_work( + &store, + (&context, Some(session_credentials(&headers))), + &LocationWorkRequest { + request_id: request.request_id, + location_id, + target, + reason: request.reason, + work: LocationWork::CheckConnection {}, + }, + ) + .await?, + })) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", + operation_id = "readConnectionTest", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("request_id" = Uuid, Path), + ("x-sqb-csrf" = String, Header)), + responses((status = 200, body = Option), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] +pub(crate) async fn result( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target, request_id)): Path<(Uuid, String, Uuid)>, + headers: HeaderMap, +) -> Result>, ApiError> { + Ok(Json( + read_person_connection_test( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, request_id), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 39a3c0f31d..f51134549f 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -6,8 +6,8 @@ use crate::api::_helpers::authentication::oidc; use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::trusted_proxy; use crate::api::_helpers::resources::{ - administrators, audit, changes, location_health, location_work, person_operations, run_reads, - runs, snapshots, trigger_reads, warehouse_metadata, + administrators, audit, changes, location_health, location_work, person_location_work, + person_operations, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] @@ -24,6 +24,8 @@ use crate::api::_helpers::resources::{ person_operations::statistics, person_operations::fingerprint, person_operations::node_status, + person_location_work::request, + person_location_work::result, password_sessions::login, password_sessions::current, password_sessions::logout, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs new file mode 100644 index 0000000000..39fd836b6e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs @@ -0,0 +1,31 @@ +//! Connection-test submission and reads share live action authority under the store lock. + +use uuid::Uuid; + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +/// No credentials means a trusted direct-store owner, never an HTTP-supplied role. +pub(crate) async fn authorize_connection_test( + transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option<(&str, &str)>), + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, AuditContext), ControlError> { + let Some(credentials) = credentials else { + return Ok((transaction, context.clone())); + }; + authorize_session( + transaction, + (context, credentials), + &PermissionScope { + action: Action::ConnectionTest, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index cadc2fcacd..2213cd9ce2 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod connection_tests; pub(crate) mod liveness; pub(crate) mod location_heartbeat; pub(crate) mod observations; diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 0cec22828b..86c2014360 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -4,6 +4,7 @@ pub mod connect_location_work; pub mod location_health; pub mod location_work; pub(crate) mod read_location_work; +pub mod read_person_connection_test; pub(crate) mod read_platform_events; pub mod report_location_health; pub mod report_remote_location_health; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs new file mode 100644 index 0000000000..a183cd81e9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs @@ -0,0 +1,32 @@ +//! Connection-test results require live action authority and never expose sensor work. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::platform::_helpers::connection_tests::authorize_connection_test; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRecord}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn read_person_connection_test( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + (location_id, target, request_id): (Uuid, &str, Uuid), +) -> Result, ControlError> { + let (transaction, _) = authorize_connection_test( + begin(store, TransactionMode::Write).await?, + (context, Some(credentials)), + (location_id, target), + ) + .await?; + let (transaction, result) = read_location_work(transaction, request_id).await?; + let result = result.map(|stored| stored.record).filter(|record| { + record.request_json.location_id == location_id + && record.request_json.target == target + && matches!(record.request_json.work, LocationWork::CheckConnection {}) + }); + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index df0c73f23d..7c3b8698af 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -1,4 +1,4 @@ -//! Only trusted store owners enqueue named work; retries preserve the original generation/deadline. +//! Named work requires a trusted owner or live person authority; retries preserve the original pin. use chrono::Duration; use serde_json::json; @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::credentials::main::append_audit::append_audit; use crate::credentials::models::AuditContext; +use crate::platform::_helpers::connection_tests::authorize_connection_test; use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; @@ -16,7 +17,7 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub async fn request_location_work( store: &ControlStore, - context: &AuditContext, + (context, credentials): (&AuditContext, Option<(&str, &str)>), request: &LocationWorkRequest, ) -> Result { if matches!(request.work, LocationWork::EvaluateSensor { .. }) { @@ -32,7 +33,12 @@ pub async fn request_location_work( "location work needs a target, reason and trusted actor".into(), )); } - let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, context) = authorize_connection_test( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (request.location_id, &request.target), + ) + .await?; let (transaction, previous) = read_location_work(transaction, request.request_id).await?; if let Some(previous) = previous { if previous.record.request_json != *request || previous.record.requested_by != context.actor @@ -68,7 +74,7 @@ pub async fn request_location_work( commit( append_audit( transaction, - context, + &context, ( Uuid::new_v5(&request.request_id, b"location-work-requested"), "location.work_requested", diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index cd79677e03..6078f48adb 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -85,6 +85,13 @@ pub struct LocationWorkRequest { pub work: LocationWork, } +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct ConnectionTestRequest { + pub request_id: Uuid, + pub reason: String, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct LocationWorkPoll { diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 074dad859a..55f6d57764 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -40,6 +40,8 @@ mod test_location_work; mod test_oidc; #[path = "test_passwords.rs"] mod test_passwords; +#[path = "test_person_location_work.rs"] +mod test_person_location_work; #[path = "test_proxy.rs"] mod test_proxy; #[path = "test_reads.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 3ded48bc65..2701c9171b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -86,7 +86,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable work: LocationWork::CheckConnection {}, }; assert!( - request_location_work(store, &audit, &request) + request_location_work(store, (&audit, None), &request) .await .is_err(), "unpublished target" @@ -99,7 +99,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable ..audit.clone() }; assert!( - request_location_work(store, &invalid_audit, &request) + request_location_work(store, (&invalid_audit, None), &request) .await .is_err() ); @@ -110,7 +110,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable .is_none() ); assert!( - request_location_work(store, &audit, &request) + request_location_work(store, (&audit, None), &request) .await .expect("enqueue") ); @@ -121,7 +121,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable input.generation = 2; publish_snapshot(store, &input).await.expect("republish"); assert!( - !request_location_work(store, &audit, &request) + !request_location_work(store, (&audit, None), &request) .await .expect("retry after redeploy") ); @@ -136,7 +136,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable ..request.clone() }; assert!( - request_location_work(store, &audit, &changed) + request_location_work(store, (&audit, None), &changed) .await .is_err() ); @@ -145,7 +145,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable ..audit.clone() }; assert!( - request_location_work(store, &changed_actor, &request) + request_location_work(store, (&changed_actor, None), &request) .await .is_err() ); @@ -295,7 +295,7 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable request_id: Uuid::now_v7(), ..request }; - request_location_work(store, &audit, &next) + request_location_work(store, (&audit, None), &next) .await .expect("another request"); execute( diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs new file mode 100644 index 0000000000..94422c144e --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -0,0 +1,183 @@ +//! Person connection tests use live built-ins and the same atomic work queue on both stores. + +use uuid::Uuid; + +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + create_principal::create_principal, login_password::login_password, set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorAuthentication, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, + PrincipalKind, RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::{ + read_person_connection_test::read_person_connection_test, + request_location_work::request_location_work, +}; +use crate::platform::models::{LocationWork, LocationWorkRequest}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_precedes_work_reads_and_writes() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite person work", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL person work", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some(std::net::Ipv4Addr::LOCALHOST.into()), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = + create_principal(store, &context, ("orders-developer", PrincipalKind::Person)) + .await + .expect("person"); + let mut sessions = Vec::new(); + for (id, name) in [(admin, "orders-admin"), (person, "orders-developer")] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid login") + }; + sessions.push(session); + } + let credentials = (sessions[1].bearer.as_str(), sessions[1].csrf.as_str()); + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let assignment = RoleAssignmentRequest { + principal_id: person, + role: BuiltinRole::Developer, + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }; + assign_builtin_role( + store, + (&context, (&sessions[0].bearer, &sessions[0].csrf)), + &assignment, + &AdministratorAuthentication::Password, + ) + .await + .expect("assign existing builtin"); + let request = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + assert!( + request_location_work(store, (&context, Some(credentials)), &request) + .await + .expect("authorized enqueue") + ); + assert!( + !request_location_work(store, (&context, Some(credentials)), &request) + .await + .expect("idempotent retry") + ); + let result = read_person_connection_test( + store, + (&context, credentials), + (location, "development", request.request_id), + ) + .await + .expect("authorized read") + .expect("record"); + assert_eq!( + result.generation, test_case.expected_generation, + "{}", + test_case.description + ); + assert_eq!(result.requested_by, format!("principal:{person}")); + assert!(result.result_json.is_none()); + for (other, target) in [(Uuid::now_v7(), "development"), (location, "preview")] { + assert!(matches!( + read_person_connection_test( + store, + (&context, credentials), + (other, target, request.request_id) + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + read_person_connection_test( + store, + (&context, (&sessions[0].bearer, &sessions[0].csrf)), + (other, target, request.request_id) + ) + .await + .expect("installation permission still filters stored scope") + .is_none() + ); + } + assign_builtin_role( + store, + (&context, (&sessions[0].bearer, &sessions[0].csrf)), + &RoleAssignmentRequest { + revoke: true, + ..assignment + }, + &AdministratorAuthentication::Password, + ) + .await + .expect("live revoke"); + assert!(matches!( + request_location_work(store, (&context, Some(credentials)), &request).await, + Err(ControlError::Denied { .. }) + )); + assert!(matches!( + read_person_connection_test( + store, + (&context, credentials), + (location, "development", request.request_id) + ) + .await, + Err(ControlError::Denied { .. }) + )); + close_test_store(fixture).await; + }); + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 2b5c264dee..db91999a0b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -202,3 +202,30 @@ def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: cookies: SimpleCookie[str] = SimpleCookie("\n".join(response.headers.get_all("Set-Cookie"))) cookie: str = "__Host-sqb-session=" + cookies["__Host-sqb-session"].value return cookie, json.load(response)["csrf_token"] + + +def verify_connection_test_transport( + api: SnapshotApi, path: str, body: dict[str, Any], administrator: tuple[str, str] +) -> None: + """Neither machine credentials, missing CSRF nor caller-supplied work cross the person API.""" + for credentials in [("", ""), (administrator[0], "wrong-csrf")]: + with pytest.raises(HTTPError) as denied: + person_request(api, path, body=body, session=credentials) + assert denied.value.code == 401 + with pytest.raises(HTTPError) as denied: + person_read(api, path + "/" + body["request_id"], session=(administrator[0], "wrong-csrf")) + assert denied.value.code == 403 + with pytest.raises(HTTPError) as denied: + urlopen( + Request( + api.url + path, + data=json.dumps(body).encode(), + headers=person_headers(("", "")) | {"Authorization": "Bearer " + api.bearer}, + ), + timeout=5, + ) + assert denied.value.code == 401 + for extra in [{"actor": "forged"}, {"work": {"kind": "evaluate_sensor"}}]: + with pytest.raises(HTTPError) as denied: + person_request(api, path, body=body | extra, session=administrator) + assert denied.value.code == 422 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py index 502613c572..12f864c08d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py @@ -2,10 +2,13 @@ import json import os +import sqlite3 import subprocess import sys +from functools import partial from pathlib import Path from typing import Any +from urllib.error import HTTPError from uuid import uuid4 import pytest @@ -22,6 +25,13 @@ wait_for_location_health, wait_for_location_work, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, + verify_connection_test_transport, +) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -156,3 +166,171 @@ def test_given_a_published_location_when_an_owner_requests_work_then_results_ret ] == [1, 2, 3] assert api.bearer not in log.read_text() + publications.read_text() assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + LocationWorkTestCase("person healthy connection", "warehouse.db", True), + LocationWorkTestCase("person failed connection", "missing/warehouse.db", False), + ], + ids=lambda case: case.description, +) +def test_given_person_roles_when_requesting_connection_tests_then_live_scope_and_real_results_hold( + test_case: LocationWorkTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + '\n[targets.development]\nschema = "main"\n', + }, + ) + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json" + ).read_text() + ) + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + administrator: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-developer" + ) + admin: tuple[str, str] = person_login(api, administrator["name"]) + session: tuple[str, str] = person_login(api, person["name"]) + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + log: Path = tmp_path / "location.log" + path: str = f"/api/control/locations/{api.location_id}/targets/development/connection-tests" + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + _ = wait_for_generation(api, "development", 1, process, log) + request: dict[str, Any] = { + "request_id": str(uuid4()), + "reason": "Check orders connectivity", + } + for operation in [ + partial(person_request, api, path, body=request, session=session), + partial(person_read, api, path + "/" + request["request_id"], session=session), + ]: + with pytest.raises(HTTPError) as denied: + operation() + assert denied.value.code == 403 + for role in fixture["connection_test_roles"][1:3]: + assert role["allowed"] is False + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": role["role"], + "location_id": api.location_id, + "target": "development", + } + assert person_request(api, "/api/admin/roles", body=assignment, session=admin) == { + "applied": True + } + for operation in [ + partial(person_request, api, path, body=request, session=session), + partial(person_read, api, path + "/" + request["request_id"], session=session), + ]: + with pytest.raises(HTTPError) as denied: + operation() + assert denied.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin + ) == {"applied": True} + scopes: dict[str, tuple[str | None, str | None]] = { + "admin": (None, None), + "developer": (api.location_id, "development"), + } + for role in fixture["connection_test_roles"][3:]: + assert role["allowed"] is True + assignment = { + "principal_id": person["principal_id"], + "role": role["role"], + "location_id": scopes[role["role"]][0], + "target": scopes[role["role"]][1], + } + assert person_request(api, "/api/admin/roles", body=assignment, session=admin) == { + "applied": True + } + request = {"request_id": str(uuid4()), "reason": "Check orders connectivity"} + assert person_request(api, path, body=request, session=session) == {"applied": True} + assert person_request(api, path, body=request, session=session) == { + "applied": False + } + result: dict[str, Any] = wait_for_location_work( + api, request["request_id"], process, log + ) + assert ( + person_read(api, path + "/" + request["request_id"], session=session) == result + ) + assert result["result_json"]["outcome"] == { + "kind": "connection", + "healthy": test_case.expected_connection, + } + assert result["requested_by"] == "principal:" + person["principal_id"] + assert result["generation"] == 1 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin + ) == {"applied": True} + assert person_request(api, "/api/admin/roles", body=assignment, session=admin) == { + "applied": True + } + for other in [ + path.replace("development", "preview"), + path.replace(api.location_id, str(uuid4())), + ]: + with pytest.raises(HTTPError) as denied: + person_request(api, other, body=request, session=session) + assert denied.value.code == 403 + with pytest.raises(HTTPError) as denied: + person_read(api, other + "/" + request["request_id"], session=session) + assert denied.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin + ) == {"applied": True} + for operation in [ + partial(person_request, api, path, body=request, session=session), + partial(person_read, api, path + "/" + request["request_id"], session=session), + ]: + with pytest.raises(HTTPError) as denied: + operation() + assert denied.value.code == 403 + verify_connection_test_transport(api, path, request, admin) + audit: list[dict[str, Any]] = person_read( + api, "/api/control/audit?limit=100", session=admin + ) + accepted: list[dict[str, Any]] = list( + filter(lambda event: event["action"] == "location.work_requested", audit) + ) + assert {event["granting_role"] for event in accepted} == {"admin", "developer"} + assert all(event["actor"] == "principal:" + person["principal_id"] for event in accepted) + assert all( + event["scope_json"] == {"location_id": api.location_id, "target": "development"} + for event in accepted + ) + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as store: + assert store.execute("SELECT count(*) FROM sqb_location_work").fetchone() == (2,) + assert store.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + assert store.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert api.bearer not in log.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() From e5d8b0d403a0b6b796dc84e94070c54456ef3225 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:36:39 +0000 Subject: [PATCH 113/138] test: verify person session enforcement across transports --- .../administrator_lifecycle.json | 7 + .../tests/api/person_sessions/helpers.rs | 140 ++++++ .../tests/api/person_sessions/test_types.rs | 18 + .../tests/api/test_contracts.rs | 6 + .../tests/api/test_person_sessions.rs | 415 ++++++++++++++++++ .../publication/person/helpers.py | 34 ++ .../publication/test_custom_sensor.py | 47 +- .../publication/test_person_journey.py | 5 +- 8 files changed, 668 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json index 6a75cf5653..93cafde23a 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -12,6 +12,13 @@ {"role": "admin", "allowed": true}, {"role": "developer", "allowed": true} ], + "session_roles": [ + {"role": null, "expected_metadata": 403, "expected_connection": 403, "expected_job": 403}, + {"role": "viewer", "expected_metadata": 200, "expected_connection": 403, "expected_job": 403}, + {"role": "operator", "expected_metadata": 200, "expected_connection": 403, "expected_job": 200}, + {"role": "developer", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200}, + {"role": "admin", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200} + ], "job_settings": [ {"full_refresh": false}, {"full_refresh": true}, diff --git a/crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs b/crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs new file mode 100644 index 0000000000..13bcf0b11d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs @@ -0,0 +1,140 @@ +//! Authenticate real browser transports and observe their durable queue effects. + +use std::collections::{BTreeMap, BTreeSet}; + +use reqwest::{Client, Method, Response, StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, OidcAuthentication, TrustedProxyAuthentication}; +use crate::api::tests::helpers::start_api_with_authentication; +use crate::api::tests::oidc_helpers::{callback, login}; +use crate::api::tests::oidc_test_types::TestProvider; +use crate::api::tests::person_sessions_test_types::BrowserSession; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) async fn browser_sessions( + store: &ControlStore, + provider: &TestProvider, + person: Uuid, +) -> Vec { + let origin = "https://control.example.com"; + let password = start_api_with_authentication( + store, + ApiAuthentication { + password_origin: Some(origin.into()), + ..Default::default() + }, + ) + .await; + let proxy = start_api_with_authentication( + store, + ApiAuthentication { + trusted_proxy: Some(TrustedProxyAuthentication { + origin: origin.into(), + trusted_peers: BTreeSet::from(["127.0.0.1".parse().expect("socket peer")]), + identities: BTreeMap::from([("orders-person".into(), person)]), + }), + ..Default::default() + }, + ) + .await; + let oidc = start_api_with_authentication( + store, + ApiAuthentication { + oidc: Some(OidcAuthentication { + origin: origin.into(), + issuer: provider.state.issuer.clone(), + client_id: "orders-control".into(), + client_secret_env: None, + certificate_authority: Some(provider.authority.clone()), + identities: BTreeMap::from([("orders-person".into(), person)]), + }), + ..Default::default() + }, + ) + .await; + let password_response = provider + .client + .post(format!("{}/api/auth/password/login", password.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, origin) + .json(&json!({"username":"orders-person", "password":"orders-password-2026"})) + .send() + .await + .expect("password login"); + let proxy_response = provider + .client + .get(format!("{}/api/auth/proxy", proxy.url)) + .header(header::HOST, "control.example.com") + .header("x-sqb-user", "orders-person") + .send() + .await + .expect("trusted proxy login"); + let flow = login(provider, &oidc.url).await; + let oidc_response = callback(provider, &oidc.url, &flow).await; + let mut sessions = Vec::new(); + for (api, response, identity_path) in [ + (password, password_response, "/identity/principal_id"), + (proxy, proxy_response, "/principal_id"), + (oidc, oidc_response, "/principal_id"), + ] { + assert_eq!(response.status(), StatusCode::OK); + let cookie = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("browser session") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let body: Value = response.json().await.expect("session response"); + assert_eq!(body.pointer(identity_path), Some(&json!(person))); + assert!(body.get("bearer").is_none()); + sessions.push(BrowserSession { + api, + cookie, + csrf: body["csrf_token"].as_str().expect("CSRF").into(), + }); + } + sessions +} + +pub(crate) async fn request( + client: &Client, + session: &BrowserSession, + (method, path): (Method, &str), + (body, csrf): (&Value, &str), +) -> Response { + client + .request(method, format!("{}{path}", session.api.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .header(header::COOKIE, &session.cookie) + .header("x-sqb-csrf", csrf) + .json(body) + .send() + .await + .expect("browser operation") +} + +pub(crate) async fn queue_counts(store: &ControlStore) -> Value { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("inspect queues"); + let (transaction, rows) = query( + transaction, + "SELECT (SELECT count(*) FROM sqb_location_work) AS work, \ + (SELECT count(*) FROM sqb_runs) AS runs, \ + (SELECT count(*) FROM sqb_write_claims) AS claims", + vec![], + ) + .await + .expect("queue counts"); + commit(transaction).await.expect("close inspection"); + json!(rows[0]) +} diff --git a/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs b/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs new file mode 100644 index 0000000000..dbef04a914 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs @@ -0,0 +1,18 @@ +//! Browser session fixtures and independent role-matrix expectations. + +use crate::api::tests::test_types::TestApi; +use crate::credentials::models::BuiltinRole; + +pub(crate) struct BrowserSession { + pub(crate) api: TestApi, + pub(crate) cookie: String, + pub(crate) csrf: String, +} + +#[derive(serde::Deserialize)] +pub(crate) struct SessionRoleCase { + pub(crate) role: Option, + pub(crate) expected_metadata: u16, + pub(crate) expected_connection: u16, + pub(crate) expected_job: u16, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 55f6d57764..31373736c1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -14,6 +14,10 @@ mod oidc_helpers; mod oidc_test_types; #[path = "output/helpers.rs"] mod output_helpers; +#[path = "person_sessions/helpers.rs"] +mod person_sessions_helpers; +#[path = "person_sessions/test_types.rs"] +mod person_sessions_test_types; #[path = "recall/helpers.rs"] mod recall_helpers; #[path = "statistics/eligibility/helpers.rs"] @@ -42,6 +46,8 @@ mod test_oidc; mod test_passwords; #[path = "test_person_location_work.rs"] mod test_person_location_work; +#[path = "test_person_sessions.rs"] +mod test_person_sessions; #[path = "test_proxy.rs"] mod test_proxy; #[path = "test_reads.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs new file mode 100644 index 0000000000..c87ae8d35a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -0,0 +1,415 @@ +//! Existing human authentication mechanisms share live authority, never machine authority. + +use reqwest::{Method, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::helpers::stop_api; +use crate::api::tests::oidc_helpers::{start_provider, stop_provider}; +use crate::api::tests::person_sessions_helpers::{browser_sessions, queue_counts, request}; +use crate::api::tests::person_sessions_test_types::SessionRoleCase; +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + create_principal::create_principal, login_password::login_password, set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorAuthentication, AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind, + RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::LocationWork; +use crate::runs::tests::helpers::{request as run_request, request_run}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_work_kind_remain_enforced() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite browser session matrix", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL browser session matrix", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some(std::net::Ipv4Addr::LOCALHOST.into()), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("administrator"); + let person = + create_principal(store, &context, ("orders-person", PrincipalKind::Person)) + .await + .expect("person"); + for id in [admin, person] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("password"); + } + let PasswordLogin::Authenticated(admin_session) = + login_password(store, &context, ("orders-admin", "orders-password-2026")) + .await + .expect("admin login") + else { + panic!("valid admin") + }; + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = + json!({"kind":"custom", "function":"orders_ready", "code_hash":"orders_v1"}); + let published = publish_snapshot(store, &input).await.expect("publication"); + assert_eq!( + published.generation, test_case.expected_generation, + "{}", + test_case.description + ); + let seed_run = run_request((location, published.snapshot_id), 100, 0, &["orders"]); + request_run(store, &seed_run) + .await + .expect("nonempty metadata control"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let evaluation = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("fenced sensor request") + .expect("requested"); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("inspect sensor"); + let (transaction, stored) = read_location_work(transaction, evaluation.evaluation_id) + .await + .expect("sensor work exists"); + assert!(matches!( + stored.expect("actual work").record.request_json.work, + LocationWork::EvaluateSensor { .. } + )); + commit(transaction).await.expect("close read"); + let provider = start_provider().await; + let sessions = browser_sessions(store, &provider, person).await; + assert_eq!(sessions.len(), 3, "password, proxy and OIDC all execute"); + let path = + format!("/api/control/locations/{location}/targets/development/connection-tests"); + let metadata = format!("/api/control/locations/{location}/targets/development/runs"); + let inputs: Value = serde_json::from_str(&std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json")).expect("DuckShop matrix")).expect("JSON"); + let roles: Vec = + serde_json::from_value(inputs["session_roles"].clone()) + .expect("independent expectations"); + let mut expected_work = 1; + let mut expected_runs = 1; + let mut last_request = Value::Null; + for role_case in roles { + for role in role_case.role.iter().copied() { + assign_builtin_role( + &replica, + (&context, (&admin_session.bearer, &admin_session.csrf)), + &RoleAssignmentRequest { + principal_id: person, + role, + location_id: None, + target: None, + revoke: false, + }, + &AdministratorAuthentication::Password, + ) + .await + .expect("assign role through independent store"); + } + for session in &sessions { + let body = + json!({"request_id":Uuid::now_v7(), "reason":"Check orders connectivity"}); + last_request = body.clone(); + let connection = request( + &provider.client, + session, + (Method::POST, &path), + (&body, &session.csrf), + ) + .await; + assert_eq!(connection.status().as_u16(), role_case.expected_connection); + assert_eq!(connection.headers()["cache-control"], "no-store"); + let read = request( + &provider.client, + session, + (Method::GET, &format!("{path}/{}", evaluation.evaluation_id)), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(read.status().as_u16(), role_case.expected_connection); + let read_body: Value = read.json().await.expect("sensor response"); + assert_eq!( + read_body.is_null(), + role_case.expected_connection == StatusCode::OK.as_u16(), + "authorized reads hide actual sensor work as null" + ); + let page = request( + &provider.client, + session, + (Method::GET, &metadata), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(page.status().as_u16(), role_case.expected_metadata); + let page: Value = page.json().await.expect("metadata response"); + assert_eq!( + page.is_array(), + role_case.expected_metadata == StatusCode::OK.as_u16() + ); + assert!(!page.to_string().contains("select 1 as id")); + let job = json!({"run_id":Uuid::now_v7(), "location_id":location, "target":"development", "job_key":"orders", "reason":"Refresh orders", "priority":0}); + assert_eq!( + request( + &provider.client, + session, + (Method::POST, "/api/control/runs"), + (&job, &session.csrf) + ) + .await + .status() + .as_u16(), + role_case.expected_job + ); + for (endpoint, request_body) in + [(&path[..], &body), ("/api/control/runs", &job)] + { + assert_eq!( + request( + &provider.client, + session, + (Method::POST, endpoint), + (request_body, "wrong-csrf") + ) + .await + .status(), + StatusCode::UNAUTHORIZED + ); + } + for machine_path in [ + format!("/api/locations/{location}/targets/development/snapshots"), + format!( + "/api/locations/{location}/work/{}", + evaluation.evaluation_id + ), + format!( + "/api/locations/{location}/targets/development/runs/{}/recall/output_tail?invocation=scheduler/{}", + seed_run.run_id, seed_run.run_id + ), + format!( + "/api/locations/{location}/targets/development/runs/{}/recall/artifacts?invocation=scheduler/{}", + seed_run.run_id, seed_run.run_id + ), + ] { + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &machine_path), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::UNAUTHORIZED + ); + } + } + expected_work += usize::from(role_case.expected_connection == 200) * sessions.len(); + expected_runs += usize::from(role_case.expected_job == 200) * sessions.len(); + assert_eq!( + queue_counts(store).await, + json!({"work":expected_work,"runs":expected_runs,"claims":0}) + ); + for role in role_case.role.iter().copied() { + assign_builtin_role( + &replica, + (&context, (&admin_session.bearer, &admin_session.csrf)), + &RoleAssignmentRequest { + principal_id: person, + role, + location_id: None, + target: None, + revoke: true, + }, + &AdministratorAuthentication::Password, + ) + .await + .expect("revoke through independent store"); + } + for session in &sessions { + assert_eq!( + request( + &provider.client, + session, + (Method::POST, &path), + (&last_request, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + ( + Method::GET, + &format!( + "{path}/{}", + last_request["request_id"].as_str().expect("request ID") + ) + ), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &metadata), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + queue_counts(store).await, + json!({"work":expected_work,"runs":expected_runs,"claims":0}) + ); + } + assign_builtin_role( + &replica, + (&context, (&admin_session.bearer, &admin_session.csrf)), + &RoleAssignmentRequest { + principal_id: person, + role: crate::credentials::models::BuiltinRole::Developer, + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }, + &AdministratorAuthentication::Password, + ) + .await + .expect("scoped developer"); + for session in &sessions { + let result = request( + &provider.client, + session, + ( + Method::GET, + &format!( + "{path}/{}", + last_request["request_id"].as_str().expect("accepted ID") + ), + ), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(result.status(), StatusCode::OK); + assert_eq!( + result + .json::() + .await + .expect("actual connection record")["request_json"]["request_id"], + last_request["request_id"] + ); + for other in [ + path.replace("development", "preview"), + path.replace(&location.to_string(), &Uuid::now_v7().to_string()), + ] { + assert_eq!( + request( + &provider.client, + session, + (Method::POST, &other), + (&last_request, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + ( + Method::GET, + &format!( + "{other}/{}", + last_request["request_id"].as_str().expect("ID") + ) + ), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &format!("{path}/{}", evaluation.evaluation_id)), + (&Value::Null, "wrong-csrf") + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + queue_counts(store).await, + json!({"work":expected_work,"runs":expected_runs,"claims":0}) + ); + for session in sessions { + stop_api(session.api).await; + } + stop_provider(provider).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index db91999a0b..273799561e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -10,6 +10,7 @@ from subprocess import Popen from typing import Any from urllib.error import HTTPError +from urllib.parse import urlencode from urllib.request import Request, urlopen import pytest @@ -61,6 +62,39 @@ def person_read(api: SnapshotApi, path: str, *, session: tuple[str, str]) -> Any return json.load(response) +def verify_compute_transport(api: SnapshotApi, session: tuple[str, str], run_id: str) -> None: + """Real retained SQL/output are readable by the location, never by a browser cookie alone.""" + metadata: list[dict[str, Any]] = person_read( + api, f"/api/control/locations/{api.location_id}/targets/development/runs", session=session + ) + assert metadata[0]["run_id"] == run_id + assert not ({"sql", "compiled_sql", "output", "rows", "evidence"} & metadata[0].keys()) + path: str = f"/api/locations/{api.location_id}/targets/development/runs/{run_id}" + headers: dict[str, str] = {"Authorization": "Bearer " + api.bearer} + with urlopen(Request(api.url + path + "/invocations", headers=headers), timeout=5) as response: + invocations: list[dict[str, Any]] = json.load(response) + assert any(row["invocation_id"].startswith("worker/") for row in invocations) + for view in ["artifacts", "output_tail"]: + evidence: list[dict[str, Any]] = [] + for invocation in invocations: + endpoint: str = ( + path + + "/recall/" + + view + + "?" + + urlencode({"invocation": invocation["invocation_id"]}) + ) + with urlopen(Request(api.url + endpoint, headers=headers), timeout=5) as response: + evidence.extend(json.load(response)) + with pytest.raises(HTTPError) as rejected: + person_read(api, endpoint, session=session) + assert rejected.value.code == 401 + assert evidence, view + assert any( + row["invocation_id"].startswith("worker/") and row["text"] for row in evidence + ), view + + def wait_for_person_catalog( api: SnapshotApi, session: tuple[str, str], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py index 1c6fbcdc4c..e6da7b8e1b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -19,9 +19,17 @@ scheduled_worker, sensor_calls, wait_for_generation, + wait_for_location_work, wait_for_run, wait_for_sensor_phase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, + verify_compute_transport, +) @pytest.mark.parametrize( @@ -91,7 +99,31 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a data.write_text(json.dumps({"mode": "failed"})) monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) monkeypatch.setenv("SQLBUILD_SENSOR_CALLS", str(calls)) - with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + with scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + administrator: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-developer" + ) + admin: tuple[str, str] = person_login(worker.api, administrator["name"]) + session: tuple[str, str] = person_login(worker.api, person["name"]) + assert person_request( + worker.api, + "/api/admin/roles", + body={ + "principal_id": person["principal_id"], + "role": "developer", + "location_id": worker.api.location_id, + "target": "development", + }, + session=admin, + ) == {"applied": True} monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") log: Path = tmp_path / "location.log" @@ -115,6 +147,17 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a _ = wait_for_generation(worker.api, "development", 2, location, log) failed: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "failed") assert failed["request"]["cursor"] == {"event_storage_id": 0} + work: dict[str, Any] = wait_for_location_work( + worker.api, failed["request"]["evaluation_id"], location, log + ) + assert work["request_json"]["work"]["kind"] == "evaluate_sensor" + assert work["request_json"]["request_id"] == failed["request"]["evaluation_id"] + path: str = ( + f"/api/control/locations/{worker.api.location_id}/targets/development/connection-tests/" + + failed["request"]["evaluation_id"] + ) + for browser in [admin, session]: + assert person_read(worker.api, path, session=browser) is None data.write_text(json.dumps({"mode": "timeout"})) timed_out: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "timed_out") assert timed_out["request"]["cursor"] == failed["request"]["cursor"] @@ -128,6 +171,8 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a worker.api, worker.process, worker.log, job_key="annual_orders" ) assert destination["status"] == test_case.expected_status, log.read_text() + for browser in [admin, session]: + verify_compute_transport(worker.api, browser, destination["run_id"]) # The next successful evaluation skips but retains its committed cursor. skipped: dict[str, Any] = wait_for_sensor_phase( worker.control_store, "completed", empty_runs=True diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index f12446d360..552ce5fcef 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -31,6 +31,7 @@ person_read, person_request, provision_person, + verify_compute_transport, verify_security_audit, wait_for_person_catalog, ) @@ -131,9 +132,7 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi assert person_request( worker.api, "/api/admin/roles", body=assignment, session=admin_session ) == {"applied": True} - metadata: list[dict[str, Any]] = person_read(worker.api, runs_path, session=session) - assert metadata[0]["run_id"] == scheduled["run_id"] - assert not ({"sql", "compiled_sql", "output", "rows", "evidence"} & metadata[0].keys()) + verify_compute_transport(worker.api, session, scheduled["run_id"]) for path, body in [ ("/api/control/runs", request), ("/api/control/runs/cancel", cancellation), From d5f51de257e0794cd747b015fab90d56c43943a1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:52:10 +0000 Subject: [PATCH 114/138] test: verify installed PostgreSQL administrator recovery --- .../publication/test_installed_recovery.py | 164 ++++++++++++++++++ .../publication/wheel/cli/helpers.py | 18 +- .../publication/wheel/helpers.py | 28 +++ 3 files changed, 204 insertions(+), 6 deletions(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py index 69f482a87b..395bcfc483 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py @@ -1,14 +1,30 @@ """An installed native executable retains offline administrator recovery guarantees.""" +import json import shutil +import subprocess from pathlib import Path +from typing import Any +import psycopg import pytest +from testcontainers.postgres import PostgresContainer +from scripts.native_tests.constants import POSTGRES_IMAGE from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( InstalledRecoveryTestCase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.api.helpers import ( + login, + request, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + serve, + successful, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers import ( + postgres_recovery_state, verify_recovery, ) @@ -30,3 +46,151 @@ def test_given_an_installed_server_when_recovered_offline_then_authority_is_rest binary: str | None = shutil.which(test_case.executable) assert binary is not None assert test_case.expected_rejection in verify_recovery(Path(binary)) + + +@pytest.mark.parametrize( + "test_case", + [ + InstalledRecoveryTestCase( + description="installed PostgreSQL recovery rolls back audit faults and reloads sessions", + executable="sqb", + expected_rejection="cannot remove or disable the final usable installation administrator", + ), + ], + ids=lambda case: case.description, +) +def test_given_postgres_servers_when_offline_recovery_fails_then_retry_revokes_only_the_recovered_person( + test_case: InstalledRecoveryTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + binary: str | None = shutil.which(test_case.executable) + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + replica: Path = tmp_path / "replica" + replica.mkdir() + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json" + ).read_text() + ) + state_sql: str = ( + "SELECT (SELECT COUNT(*) FROM sqb_principals WHERE disabled_at IS NOT NULL)," + "(SELECT COUNT(*) FROM sqb_role_assignments WHERE revoked_at IS NULL)," + "(SELECT COUNT(*) FROM sqb_password_sessions WHERE revoked_at IS NULL)," + "(SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.offline_recovered')," + "(SELECT revision FROM sqb_authentication_revision WHERE singleton=1)" + ) + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + url: str = postgres.get_connection_url() + extra: dict[str, str] = {"SQLBUILD_CONTROL_DATABASE_URL": url} + people: list[dict[str, Any]] = [] + for command, name in [ + ("create-user", fixture["first_person"]), + ("create-principal", fixture["second_person"]), + ]: + person: dict[str, Any] = successful( + administer(installation, [command, name], extra_environment=extra) + ) + successful( + administer( + installation, + ["set-password", person["principal_id"]], + fixture["password"], + extra_environment=extra, + ) + ) + people.append(person) + principal: str = people[0]["principal_id"] + recovery: list[str] = ["recover-user", principal, "--reason", fixture["recovery_reason"]] + with ( + serve(installation, extra_environment=extra) as first, + serve((Path(binary), replica), extra_environment=extra) as second, + ): + previous: tuple[str, str] = login(first, people[0]["name"], fixture["password"]) + other: tuple[str, str] = login(second, people[1]["name"], fixture["password"]) + for endpoint in [first, second]: + assert request(endpoint + "/api/auth/session", session=previous)[0] == 200 + assert request(endpoint + "/api/auth/session", session=other)[0] == 200 + rejected: subprocess.CompletedProcess[str] = administer( + installation, recovery, extra_environment=extra + ) + assert rejected.returncode != 0 and not rejected.stdout + assert "offline recovery requires exclusive control-store access" in rejected.stderr + with psycopg.connect(url) as control: + control.execute( + "UPDATE sqb_principals SET disabled_at=now() WHERE principal_id=%s", (principal,) + ) + control.execute( + "UPDATE sqb_role_assignments SET revoked_at=now(),active_key=NULL " + "WHERE principal_id=%s", + (principal,), + ) + control.execute( + "CREATE FUNCTION reject_recovery_audit() RETURNS trigger LANGUAGE plpgsql AS $$ " + "BEGIN RAISE EXCEPTION 'recovery audit unavailable'; END; $$; " + "CREATE TRIGGER reject_recovery_audit BEFORE INSERT ON sqb_security_audit " + "FOR EACH ROW WHEN (NEW.action='admin.offline_recovered') " + "EXECUTE FUNCTION reject_recovery_audit()" + ) + before: Any = control.execute(state_sql).fetchone() + assert before is not None + assert before[:4] == (1, 0, 2, 0) + retained_before: dict[str, str] = postgres_recovery_state(control) + rejected = administer(installation, recovery, extra_environment=extra) + assert rejected.returncode != 0 and not rejected.stdout + assert "recovery audit unavailable" in rejected.stderr + with psycopg.connect(url) as control: + assert control.execute(state_sql).fetchone() == before + assert postgres_recovery_state(control) == retained_before + control.execute("DROP TRIGGER reject_recovery_audit ON sqb_security_audit") + control.execute("DROP FUNCTION reject_recovery_audit()") + assert successful(administer(installation, recovery, extra_environment=extra)) == { + "principal_id": principal, + "role": "admin", + "recovered": True, + } + with psycopg.connect(url) as control: + after: Any = control.execute(state_sql).fetchone() + assert after is not None + assert after[:4] == (0, 1, 1, 1) + assert after[4] > before[4] + assert control.execute("SELECT COUNT(*) FROM sqb_admin_initialization").fetchone() == ( + 1, + ) + retained_after: dict[str, str] = postgres_recovery_state(control) + for table in ("sqb_installation", "sqb_admin_initialization"): + assert retained_after[table] == retained_before[table] + audit: Any = control.execute( + "SELECT scope_json,after_json FROM sqb_security_audit " + "WHERE action='admin.offline_recovered'" + ).fetchone() + assert audit is not None + assert audit[0] == {"principal_id": principal} + assert audit[1]["reason"] == fixture["recovery_reason"] + with ( + serve(installation, extra_environment=extra) as first, + serve((Path(binary), replica), extra_environment=extra) as second, + ): + current: tuple[str, str] = login(first, people[0]["name"], fixture["password"]) + for endpoint in [first, second]: + assert request(endpoint + "/api/auth/session", session=previous)[0] == 401 + assert request(endpoint + "/api/auth/session", session=other)[0] == 200 + assert ( + request( + endpoint + "/api/admin/people/" + principal, + b'"revoke"', + (current[0], "wrong-csrf"), + )[0] + == 401 + ) + assert request(endpoint + "/api/auth/session", session=current)[0] == 200 + status, _, body = request( + endpoint + "/api/admin/people/" + principal, b'"revoke"', current + ) + assert status == 403 + assert test_case.expected_rejection in body["error"] + rejected = administer( + installation, ["create-user", "replacement-admin"], extra_environment=extra + ) + assert rejected.returncode != 0 and not rejected.stdout + assert "bootstrap is closed" in rejected.stderr diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py index f9be256afb..11785d7dae 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py @@ -10,22 +10,26 @@ from typing import Any -def environment() -> dict[str, str]: +def environment(extra: dict[str, str] | None = None) -> dict[str, str]: """Never inherit a real control store into a disposable acceptance run.""" values: dict[str, str] = os.environ.copy() values.pop("SQLBUILD_CONTROL_DATABASE_URL", None) - return values + return values | (extra or {}) def administer( - installation: tuple[Path, Path], arguments: list[str], password: str | None = None + installation: tuple[Path, Path], + arguments: list[str], + password: str | None = None, + *, + extra_environment: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: """Password input remains on stdin and is never included in process arguments or logs.""" binary, root = installation return subprocess.run( [str(binary), "admin", *arguments], cwd=root, - env=environment(), + env=environment(extra_environment), input=password, text=True, encoding="utf-8", @@ -42,7 +46,9 @@ def successful(output: subprocess.CompletedProcess[str]) -> dict[str, Any]: @contextmanager -def serve(installation: tuple[Path, Path]) -> Iterator[str]: +def serve( + installation: tuple[Path, Path], *, extra_environment: dict[str, str] | None = None +) -> Iterator[str]: """Kill and reap the server before returning, on Windows as well as Unix.""" binary, root = installation log: Path = root / "server.log" @@ -58,7 +64,7 @@ def serve(installation: tuple[Path, Path]) -> Iterator[str]: "https://control.example.com", ], cwd=root, - env=environment(), + env=environment(extra_environment), stdout=subprocess.DEVNULL, stderr=stderr, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py index 5d2df6f246..e94c59a59b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py @@ -1,5 +1,7 @@ """Bootstrap, authenticate, lose authority, recover offline and authenticate again.""" +import hashlib +import json import sqlite3 import subprocess import sys @@ -19,6 +21,32 @@ ) +def postgres_recovery_state(connection: Any) -> dict[str, str]: + """Compare every retained security row without logging credential hashes.""" + tables: tuple[str, ...] = ( + "sqb_principals", + "sqb_role_assignments", + "sqb_password_credentials", + "sqb_password_sessions", + "sqb_external_sessions", + "sqb_authentication_revision", + "sqb_security_audit", + "sqb_admin_initialization", + "sqb_installation", + ) + return { + table: hashlib.sha256( + json.dumps( + connection.execute( + f"SELECT to_jsonb(state) FROM {table} AS state ORDER BY to_jsonb(state)::text" + ).fetchall(), + sort_keys=True, + ).encode() + ).hexdigest() + for table in tables + } + + def verify_recovery(binary: Path) -> str: """Exercise the wheel's native executable using only the Python standard library.""" binary = binary.resolve(strict=True) From ebfc652086314752bbf254c357589c3439943037 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:24:30 +0000 Subject: [PATCH 115/138] feat: add internal projection replay owners Reconstruct current publications, jobs, run outcomes and structural catalog from retained sources under serialized writer transactions. Keep security, operational ownership and source facts intact, with paged causal replay and atomic failure handling. Exercise SQLite and PostgreSQL corruption repair, protected-row equality, idempotence, rollback and concurrent readers/writers. Preserve recorded times across backward clock steps. Keep the incomplete full rebuild command unexposed. --- .../orchestration/projection_replay.json | 13 ++ .../src/declarations/_helpers/publication.rs | 2 +- .../src/declarations/main/mod.rs | 1 + .../src/declarations/main/rebuild_jobs.rs | 18 ++ .../src/runs/_helpers/lifecycle/mod.rs | 3 + .../src/runs/_helpers/lifecycle/projection.rs | 71 ++++--- .../src/runs/_helpers/lifecycle/queued.rs | 45 ++++ .../src/runs/_helpers/lifecycle/rebuild.rs | 42 ++++ .../src/runs/_helpers/lifecycle/replay.rs | 88 ++++++++ .../src/runs/main/queue_run.rs | 29 +-- .../sqlbuild-orchestration/src/runs/models.rs | 11 + .../src/snapshots/_helpers/mod.rs | 1 + .../src/snapshots/_helpers/rebuild.rs | 62 ++++++ .../warehouse_metadata/_helpers/catalog.rs | 79 ++++++- .../warehouse_metadata/main/ingest_catalog.rs | 13 +- .../src/warehouse_metadata/models.rs | 18 ++ .../tests/api/test_contracts.rs | 2 + .../api/warehouse_metadata/catalog/helpers.rs | 46 ++++- .../catalog/replay/helpers.rs | 84 ++++++++ .../tests/runs/replay/helpers.rs | 54 +++++ .../tests/runs/test_checks.rs | 1 + .../tests/runs/test_contracts.rs | 4 + .../tests/runs/test_groups.rs | 1 + .../tests/runs/test_lifecycle.rs | 3 + .../tests/runs/test_replay.rs | 193 ++++++++++++++++++ .../tests/runs/test_reruns.rs | 1 + .../tests/runs/test_types.rs | 6 + .../tests/snapshots/helpers.rs | 25 ++- .../tests/snapshots/test_contracts.rs | 4 + .../tests/snapshots/test_replay.rs | 182 +++++++++++++++++ .../tests/snapshots/test_replay_writers.rs | 129 ++++++++++++ .../tests/store/logical_state/helpers.rs | 42 ++++ .../tests/store/test_contracts.rs | 2 + 33 files changed, 1207 insertions(+), 68 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/projection_replay.json create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/rebuild_jobs.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/replay.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/replay/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_replay.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json b/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json new file mode 100644 index 0000000000..f782b7aa99 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json @@ -0,0 +1,13 @@ +{ + "snapshot_damage": [ + "DELETE FROM sqb_current_snapshots", + "UPDATE sqb_job_nodes SET node_key=node_key || '_damaged'", + "UPDATE sqb_jobs SET definition_json='{}',overlap_policy='allow'", + "INSERT INTO sqb_jobs SELECT location_id,'unpublished',job_key,snapshot_id,kind,definition_json,overlap_policy FROM sqb_jobs WHERE job_key='orders' LIMIT 1", + "INSERT INTO sqb_job_nodes SELECT location_id,target,job_key,'extra_node' FROM sqb_jobs WHERE target='unpublished'" + ], + "run_damage": [ + "DELETE FROM sqb_run_nodes", + "UPDATE sqb_runs SET status='queued',node_counts_json='{}',request_json='{}',priority=-99,status_reason='damaged',started_at=NULL,ended_at=NULL,cancel_requested_at=NULL,last_event_storage_id=0" + ] +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs index 69f65255bc..45f8ad5de8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs @@ -62,7 +62,7 @@ pub(crate) async fn publish_projections( Ok(transaction) } -async fn publish_jobs( +pub(crate) async fn publish_jobs( mut transaction: ControlTransaction, snapshot: &SnapshotRecord, definitions: &Definitions, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index 23cab85fba..ec650733ce 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -6,4 +6,5 @@ pub mod list_trigger_events; pub mod list_triggers; pub mod pause_development_triggers; pub(crate) mod publish_definitions; +pub(crate) mod rebuild_jobs; pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/rebuild_jobs.rs b/crates/sqlbuild-orchestration/src/declarations/main/rebuild_jobs.rs new file mode 100644 index 0000000000..3b44e9f90b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/rebuild_jobs.rs @@ -0,0 +1,18 @@ +//! Restore only resolved job projections from immutable publication documents. + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::_helpers::publication::publish_jobs; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn rebuild_jobs( + transaction: ControlTransaction, + snapshot: &SnapshotRecord, +) -> Result { + let (transaction, documents) = + read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["jobs", "groups"])) + .await?; + publish_jobs(transaction, snapshot, &decode_definitions(&documents)?).await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 1d0d31e732..c938ed162d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -7,5 +7,8 @@ pub(in crate::runs) mod nodes; pub(in crate::runs) mod output; pub(in crate::runs) mod projection; pub(in crate::runs) mod publish_change; +pub(in crate::runs) mod queued; pub(in crate::runs) mod reads; +pub(in crate::runs) mod rebuild; pub(in crate::runs) mod recall; +pub(in crate::runs) mod replay; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index a386ad067b..084bf8205d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -12,32 +12,63 @@ use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; pub(crate) async fn project_change( + transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + if matches!( + request.change, + RunChange::CheckStarted { .. } | RunChange::CheckFinished { .. } + ) { + return check_execution_history(transaction, request).await; + } + let mut transaction = project_run_state(transaction, request, state).await?; + if let RunChange::Launched { container_id } = &request.change { + (transaction, _) = query( + transaction, + "UPDATE sqb_run_liveness SET container_id=$1 WHERE run_id=$2", + vec![ + Parameter::OptionalText(container_id.clone()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + if matches!(request.change, RunChange::Finished { .. }) { + for table in [ + "sqb_write_claims", + "sqb_concurrency_slots", + "sqb_run_liveness", + ] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE run_id=$1"), + vec![Parameter::Id(request.run_id)], + ) + .await?; + } + } + Ok(transaction) +} + +/// Reconstruct derived status only, without releasing or recreating live physical ownership. +pub(crate) async fn project_run_state( mut transaction: ControlTransaction, request: &RunChangeRequest, state: &ChangingRun, ) -> Result { let now = transaction.now; match &request.change { - RunChange::LaunchFailed { .. } => {} - RunChange::CheckStarted { .. } | RunChange::CheckFinished { .. } => { - return check_execution_history(transaction, request).await; - } + RunChange::LaunchFailed { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } => {} RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { return project_node(transaction, request, &state.node_counts_json).await; } - RunChange::Launched { container_id } => { + RunChange::Launched { .. } => { (transaction, _) = query(transaction, "UPDATE sqb_runs SET status=CASE WHEN status='queued' THEN 'launched' ELSE status END, launched_at=COALESCE(launched_at,$1) WHERE run_id=$2", vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; - (transaction, _) = query( - transaction, - "UPDATE sqb_run_liveness SET container_id=$1 WHERE run_id=$2", - vec![ - Parameter::OptionalText(container_id.clone()), - Parameter::Id(request.run_id), - ], - ) - .await?; } RunChange::Started => { (transaction, _) = query(transaction, @@ -102,18 +133,6 @@ pub(crate) async fn project_change( ], ) .await?; - for table in [ - "sqb_write_claims", - "sqb_concurrency_slots", - "sqb_run_liveness", - ] { - (transaction, _) = query( - transaction, - &format!("DELETE FROM {table} WHERE run_id=$1"), - vec![Parameter::Id(request.run_id)], - ) - .await?; - } } } Ok(transaction) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs new file mode 100644 index 0000000000..9310c02dad --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs @@ -0,0 +1,45 @@ +//! Initial run and node projections shared by fact publication and replay. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_queued_run( + mut transaction: ControlTransaction, + request: &RunRequest, + cursor: i64, +) -> Result { + let now = transaction.now; + let selection = serde_json::to_value(&request.selection) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex( + &serde_json::to_vec(&request.write_set) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ + requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21) \ + ON CONFLICT (run_id) DO UPDATE SET location_id=excluded.location_id,target=excluded.target,snapshot_id=excluded.snapshot_id, \ + image_id=excluded.image_id,kind=excluded.kind,origin=excluded.origin,job_key=excluded.job_key,parent_run_id=excluded.parent_run_id, \ + root_run_id=excluded.root_run_id,trigger_id=excluded.trigger_id,tick_id=excluded.tick_id,rerun_of_run_id=excluded.rerun_of_run_id, \ + requested_by=excluded.requested_by,priority=excluded.priority,selection_json=excluded.selection_json,write_set_digest=excluded.write_set_digest, \ + status=excluded.status,queued_at=excluded.queued_at,node_counts_json=excluded.node_counts_json,last_event_storage_id=excluded.last_event_storage_id, \ + request_json=excluded.request_json,rewrite_history=FALSE,status_reason=NULL,launched_at=NULL,started_at=NULL,ended_at=NULL,cancel_requested_at=NULL", + vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), + Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), + Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), + Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), + Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), + Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; + for node in &request.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_nodes (run_id, node_key, attempt, status, writes) VALUES ($1,$2,1,'pending',$3)", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone()), Parameter::Boolean(request.write_set.contains(node))]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs new file mode 100644 index 0000000000..fbf5829aa5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs @@ -0,0 +1,42 @@ +//! Reconstruct run metadata and node outcomes on one serialized, atomic source cut. + +use crate::runs::_helpers::lifecycle::replay::replay_run_event; +use crate::runs::models::RunReplayRow; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Internal owner only; liveness, claims, tokens, requests, audits and Recall remain untouched. +#[cfg_attr( + not(test), + expect(dead_code, reason = "awaiting the complete rebuild command") +)] +pub(crate) async fn rebuild_run_projections(store: &ControlStore) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, unsupported) = query(transaction, + "SELECT r.run_id FROM sqb_runs r WHERE NOT EXISTS (SELECT 1 FROM recall_events e \ + WHERE e.producer='scheduler' AND e.event_type='run_queued' AND e.run_id=CAST(r.run_id AS TEXT) \ + AND e.invocation_id='scheduler/' || e.run_id) LIMIT 1", vec![]).await?; + if !unsupported.is_empty() { + return Err(ControlError::Invalid( + "run projection has no retained queued source fact".into(), + )); + } + let mut after = 0; + loop { + let rows; + (transaction, rows) = query(transaction, + "SELECT storage_id,event_schema_version AS schema_version,occurred_at,event_type,run_id,payload FROM recall_events \ + WHERE producer='scheduler' AND invocation_id='scheduler/' || run_id AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", vec![Parameter::Integer(after)]).await?; + if rows.is_empty() { + break; + } + for row in rows { + let row: RunReplayRow = decode_row(row)?; + after = row.storage_id; + transaction = replay_run_event(transaction, row).await?; + } + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/replay.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/replay.rs new file mode 100644 index 0000000000..d983660174 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/replay.rs @@ -0,0 +1,88 @@ +//! Project retained facts without republishing events or touching live ownership. + +use crate::runs::_helpers::lifecycle::projection::project_run_state; +use crate::runs::_helpers::lifecycle::queued::project_queued_run; +use crate::runs::models::{ + ChangingRun, RunChange, RunChangeRequest, RunClaim, RunReplayRow, RunRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn replay_run_event( + mut transaction: ControlTransaction, + row: RunReplayRow, +) -> Result { + if row.schema_version != 1 { + return Err(ControlError::Invalid( + "cannot replay an unsupported run fact schema".into(), + )); + } + transaction.now = row.occurred_at; + match row.event_type.as_str() { + "run_requested" => return Ok(transaction), + "run_queued" => { + let request: RunRequest = serde_json::from_value(row.payload["request"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.run_id != row.run_id { + return Err(ControlError::Invalid( + "queued fact has a different run identity".into(), + )); + } + (transaction, _) = query( + transaction, + "DELETE FROM sqb_run_nodes WHERE run_id=$1", + vec![Parameter::Id(row.run_id)], + ) + .await?; + transaction = project_queued_run(transaction, &request, row.storage_id).await?; + } + "run_claimed" => { + let claim: RunClaim = serde_json::from_value(row.payload["claim"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if claim.request.run_id != row.run_id { + return Err(ControlError::Invalid( + "claim fact has a different run identity".into(), + )); + } + } + _ => { + let request: RunChangeRequest = + serde_json::from_value(row.payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.run_id != row.run_id || request.change.event_type() != row.event_type { + return Err(ControlError::Invalid( + "run fact identity or change type does not match".into(), + )); + } + if matches!(request.change, RunChange::LaunchFailed { .. }) { + return Ok(transaction); + } + let states; + (transaction, states) = query(transaction, + "SELECT status,cancel_requested_at,node_counts_json,NULL AS worker_instance FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(row.run_id)]).await?; + let state: ChangingRun = decode_row(states.into_iter().next().ok_or_else(|| { + ControlError::Invalid("run change has no retained queued request".into()) + })?)?; + transaction = project_run_state(transaction, &request, &state).await?; + } + } + let affected; + (transaction, affected) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2 RETURNING run_id", + vec![ + Parameter::Integer(row.storage_id), + Parameter::Id(row.run_id), + ], + ) + .await?; + if affected.is_empty() { + return Err(ControlError::Invalid( + "run fact has no retained queued request".into(), + )); + } + transaction.changes.insert(Change::Run(row.run_id)); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs index fb4d86c590..f2d0ee7c95 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -1,15 +1,14 @@ //! Publish request facts, queued facts and initial node projections in the caller's transaction. use serde_json::json; -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::lifecycle::queued::project_queued_run; use crate::runs::_helpers::requests::job::frozen_group_children; use crate::runs::models::{RunKind, RunRequest}; use crate::store::errors::ControlError; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::ControlTransaction; pub(crate) async fn queue_run( transaction: ControlTransaction, @@ -30,7 +29,6 @@ async fn queue_one( mut transaction: ControlTransaction, request: &RunRequest, ) -> Result { - let now = transaction.now; let mut cursor = None; for kind in ["run_requested", "run_queued"] { let storage_id; @@ -48,26 +46,5 @@ async fn queue_one( } let cursor = cursor.ok_or_else(|| ControlError::Invalid("queued event has no storage cursor".into()))?; - let selection = serde_json::to_value(&request.selection) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - let digest = sha256_hex( - &serde_json::to_vec(&request.write_set) - .map_err(|error| ControlError::Invalid(error.to_string()))?, - ); - (transaction, _) = query(transaction, - "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ - requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21)", - vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), - Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), - Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), - Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), - Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), - Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; - for node in &request.selection { - (transaction, _) = query(transaction, - "INSERT INTO sqb_run_nodes (run_id, node_key, attempt, status, writes) VALUES ($1,$2,1,'pending',$3)", - vec![Parameter::Id(request.run_id), Parameter::Text(node.clone()), Parameter::Boolean(request.write_set.contains(node))]).await?; - } - Ok(transaction) + project_queued_run(transaction, request, cursor).await } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index a6b13e74ee..bf9f99ee28 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -448,3 +448,14 @@ pub struct RunObservation { pub last_heartbeat_at: Option>, pub observed_at: DateTime, } + +/// Retained causal order and original timestamps, never reconstructed from wall-clock sorting. +#[derive(Debug, Deserialize)] +pub(crate) struct RunReplayRow { + pub storage_id: i64, + pub schema_version: i64, + pub occurred_at: DateTime, + pub event_type: String, + pub run_id: Uuid, + pub payload: serde_json::Value, +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs index 8853fb96b6..344b06b915 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod content; pub(crate) mod preparation; pub(crate) mod publication; pub(crate) mod reading; +pub(crate) mod rebuild; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs new file mode 100644 index 0000000000..7bccf62192 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs @@ -0,0 +1,62 @@ +//! Restore current publication pointers and jobs without republishing or touching trigger state. + +use crate::declarations::main::rebuild_jobs::rebuild_jobs; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +/// Internal replay owner; current pointers and jobs change, never trigger history. +#[cfg_attr( + not(test), + expect(dead_code, reason = "awaiting the complete rebuild command") +)] +pub(crate) async fn rebuild_snapshot_projections(store: &ControlStore) -> Result<(), ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut cursor = None; + loop { + let (after, parameters) = cursor.as_ref().map_or_else( + || ("", vec![]), + |(location, target): &(uuid::Uuid, String)| { + ( + " AND (s.location_id>$1 OR (s.location_id=$1 AND s.target>$2))", + vec![Parameter::Id(*location), Parameter::Text(target.clone())], + ) + }, + ); + let rows; + (transaction, rows) = query(transaction, + &format!("SELECT s.snapshot_id,s.location_id,s.target,s.generation,s.manifest_digest,s.published_at,s.snapshot_schema_version \ + FROM sqb_snapshots s WHERE NOT EXISTS (SELECT 1 FROM sqb_snapshots newer WHERE newer.location_id=s.location_id \ + AND newer.target=s.target AND newer.generation>s.generation){after} ORDER BY s.location_id,s.target LIMIT 100"), + parameters).await?; + if rows.is_empty() { + break; + } + for row in rows { + if row["snapshot_schema_version"].as_i64() != Some(1) { + return Err(ControlError::Invalid( + "cannot replay an unsupported snapshot schema".into(), + )); + } + let snapshot: SnapshotRecord = decode_row(row)?; + transaction = rebuild_jobs(transaction, &snapshot).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_current_snapshots (location_id,target,snapshot_id,since) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (location_id,target) DO UPDATE SET snapshot_id=excluded.snapshot_id,since=excluded.since", + vec![Parameter::Id(snapshot.location_id),Parameter::Text(snapshot.target.clone()), + Parameter::Id(snapshot.snapshot_id),Parameter::Time(snapshot.published_at)]).await?; + transaction.changes.insert(Change::Snapshot( + snapshot.location_id, + snapshot.target.clone(), + )); + cursor = Some((snapshot.location_id, snapshot.target)); + } + } + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query(transaction, + &format!("DELETE FROM {table} WHERE NOT EXISTS (SELECT 1 FROM sqb_snapshots s WHERE s.location_id={table}.location_id AND s.target={table}.target)"), + vec![]).await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs index 49509fa406..c95b8ab3a9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs @@ -7,12 +7,15 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; -use crate::store::main::query::query; -use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ + Change, ControlEvent, ControlStore, ControlTransaction, Parameter, TransactionMode, +}; use crate::warehouse_metadata::_helpers::ingestion::validate_fields; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::models::{ - CatalogBatch, CatalogObservation, CatalogPage, CatalogQuery, WarehouseReconciliationCursor, + CatalogBatch, CatalogObservation, CatalogPage, CatalogQuery, CatalogReplayRow, + WarehouseCatalogRecord, WarehouseReconciliationCursor, }; pub(crate) fn validate_catalog(batch: &CatalogBatch) -> Result<(), ControlError> { @@ -73,6 +76,23 @@ pub(crate) async fn append_catalog_fact( Ok(transaction) } +pub(crate) async fn project_catalog_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + relation_id: Uuid, + record: &WarehouseCatalogRecord, +) -> Result { + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_warehouse_catalog (event_id,location_id,target,state_database,state_schema,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$6) ON CONFLICT (event_id) DO UPDATE SET \ + observed_at=excluded.observed_at,record_json=excluded.record_json,received_at=excluded.received_at", + vec![Parameter::Id(relation_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Time(now), Parameter::Json(json!(record))]).await?; + Ok(transaction) +} + pub(crate) async fn read_catalog_page( transaction: ControlTransaction, (location_id, target): (Uuid, &str), @@ -143,3 +163,56 @@ pub(crate) async fn read_catalog_page( }, )) } + +/// Internal replay owner; scan receipts, reader sessions and authoritative facts are preserved. +#[cfg_attr( + not(test), + expect(dead_code, reason = "awaiting the complete rebuild command") +)] +pub(crate) async fn rebuild_catalog_projection(store: &ControlStore) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, _) = + query(transaction, "DELETE FROM sqb_warehouse_catalog", vec![]).await?; + let mut after = 0; + loop { + let rows; + (transaction, rows) = query(transaction, + "SELECT storage_id,event_schema_version AS schema_version,occurred_at,payload FROM recall_events \ + WHERE producer='warehouse' AND event_type='warehouse_catalog_changed' AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", vec![Parameter::Integer(after)]).await?; + if rows.is_empty() { + break; + } + for row in rows { + let row: CatalogReplayRow = decode_row(row)?; + if row.schema_version != 1 { + return Err(ControlError::Invalid( + "cannot replay an unsupported catalog fact schema".into(), + )); + } + after = row.storage_id; + transaction.now = row.occurred_at; + let fact = row.payload; + if fact.present { + transaction = project_catalog_record( + transaction, + (fact.location_id, &fact.target), + fact.relation_id, + &fact.record, + ) + .await?; + } else { + (transaction, _) = query( + transaction, + "DELETE FROM sqb_warehouse_catalog WHERE event_id=$1", + vec![Parameter::Id(fact.relation_id)], + ) + .await?; + } + transaction + .changes + .insert(Change::NodeStatus(fact.location_id, fact.target)); + } + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs index 5a50d6c93e..225acd378a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs @@ -6,7 +6,9 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter}; -use crate::warehouse_metadata::_helpers::catalog::{append_catalog_fact, validate_catalog}; +use crate::warehouse_metadata::_helpers::catalog::{ + append_catalog_fact, project_catalog_record, validate_catalog, +}; use crate::warehouse_metadata::_helpers::ingestion::{ begin_ingestion, commit_ingestion, scoped_event_id, }; @@ -63,14 +65,7 @@ pub async fn ingest_catalog( (record_json.clone(), true), ) .await?; - let now = transaction.now; - (transaction, _) = query(transaction, - "INSERT INTO sqb_warehouse_catalog (event_id,location_id,target,state_database,state_schema,observed_at,record_json,received_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$6) ON CONFLICT (event_id) DO UPDATE SET \ - observed_at=excluded.observed_at,record_json=excluded.record_json,received_at=excluded.received_at", - vec![Parameter::Id(relation_id), Parameter::Id(scope.0), Parameter::Text(scope.1.into()), - Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), - Parameter::Time(now), Parameter::Json(record_json)]).await?; + transaction = project_catalog_record(transaction, scope, relation_id, record).await?; applied = true; } } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index e1b3f88f8c..98e8a43faf 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -512,6 +512,24 @@ pub struct WarehouseCatalogRecord { pub columns: Vec, } +/// Payload of the retained structural-change fact, including authoritative removals. +#[derive(Debug, Deserialize)] +pub(crate) struct CatalogProjectionFact { + pub location_id: Uuid, + pub target: String, + pub relation_id: Uuid, + pub record: WarehouseCatalogRecord, + pub present: bool, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct CatalogReplayRow { + pub storage_id: i64, + pub schema_version: i64, + pub occurred_at: DateTime, + pub payload: CatalogProjectionFact, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct CatalogBatch { diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 31373736c1..62dde71a90 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -62,6 +62,8 @@ mod test_warehouse_metadata; mod test_worker; #[path = "warehouse_metadata/catalog/helpers.rs"] mod warehouse_catalog_helpers; +#[path = "warehouse_metadata/catalog/replay/helpers.rs"] +pub(crate) mod warehouse_catalog_replay; #[path = "warehouse_metadata/events/helpers.rs"] mod warehouse_events_helpers; #[path = "warehouse_metadata/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs index fad9b3cffc..0dfe838ac2 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs @@ -9,6 +9,9 @@ use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::close_store; +use crate::warehouse_metadata::_helpers::catalog::rebuild_catalog_projection; use crate::warehouse_metadata::models::CatalogBatch; pub(crate) async fn catalog_history( @@ -42,6 +45,7 @@ pub(crate) async fn catalog_history( ); let initial = evidence(store).await; assert_eq!(initial["facts"].as_array().expect("facts").len(), 1); + crate::api::tests::warehouse_catalog_replay::catalog_replay_contract(store).await; for (field, value, status) in [ ("generation", json!(2), StatusCode::CONFLICT), ( @@ -107,16 +111,53 @@ pub(crate) async fn catalog_history( "fact, projection and receipt roll back together" ); } + let replica = second_store(store).await; + let transaction = begin(store, TransactionMode::Write) + .await + .expect("held catalog writer"); + let repair_store = replica.clone(); + let mut repair = tokio::spawn(async move { rebuild_catalog_projection(&repair_store).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut repair) + .await + .is_err() + ); + let publisher = outbound.clone(); + let typed: CatalogBatch = serde_json::from_value(batch.clone()).expect("changed catalog batch"); + let mut publication = + tokio::spawn(async move { publisher.sync_catalog("development", &typed).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut publication) + .await + .is_err() + ); assert_eq!( - post((client, &url, bearer), &batch, StatusCode::OK).await["applied"], - true + evidence(&replica).await, + initial, + "reader retains the previous committed cut" + ); + commit(transaction).await.expect("release catalog writer"); + tokio::time::timeout(std::time::Duration::from_secs(20), repair) + .await + .expect("catalog replay deadline") + .expect("join replay") + .expect("serialized replay"); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(20), publication) + .await + .expect("catalog publication deadline") + .expect("join publication") + .expect("serialized publication") + .applied ); + close_store(replica).await; let changed = evidence(store).await; assert_eq!(changed["facts"].as_array().expect("facts").len(), 2); assert_eq!( changed["current"][0]["record_json"]["columns"][1]["data_type"], "INTEGER" ); + crate::api::tests::warehouse_catalog_replay::catalog_replay_contract(store).await; let mut obsolete: Vec = Vec::new(); for index in 0..205 { let mut item = record.clone(); @@ -189,6 +230,7 @@ pub(crate) async fn catalog_history( removed, "committed final-page replay appends no facts" ); + crate::api::tests::warehouse_catalog_replay::catalog_replay_contract(store).await; } async fn post( diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/replay/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/replay/helpers.rs new file mode 100644 index 0000000000..7107789987 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/replay/helpers.rs @@ -0,0 +1,84 @@ +//! Repair and rollback against facts produced by real authenticated catalog HTTP journeys. + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; +use crate::warehouse_metadata::_helpers::catalog::rebuild_catalog_projection; + +pub(crate) async fn catalog_replay_contract(store: &ControlStore) { + let expected = logical_state(store).await; + let transaction = begin(store, TransactionMode::Write) + .await + .expect("damage cut"); + let (transaction, _) = query(transaction, "DELETE FROM sqb_warehouse_catalog", vec![]) + .await + .expect("missing catalog"); + commit(transaction).await.expect("damage commit"); + let damaged = logical_state(store).await; + let fault = TransactionFault { + description: "catalog replay rollback after projection clear", + table: "sqb_warehouse_catalog", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected catalog replay failure')", + postgres_body: "RAISE EXCEPTION 'injected catalog replay failure'", + }; + install_fault(store, &fault).await; + let error = rebuild_catalog_projection(store) + .await + .expect_err("injected fault"); + assert!( + error + .to_string() + .contains("injected catalog replay failure"), + "{error}" + ); + assert_eq!(logical_state(store).await, damaged); + remove_fault(store, &fault).await; + rebuild_catalog_projection(store) + .await + .expect("repair catalog"); + assert_eq!(logical_state(store).await, expected); + rebuild_catalog_projection(store) + .await + .expect("idempotent repair"); + assert_eq!(logical_state(store).await, expected); + + let transaction = begin(store, TransactionMode::Write) + .await + .expect("source fault"); + let (transaction, _) = query( + transaction, + "UPDATE recall_events SET event_schema_version=999 WHERE event_type='warehouse_catalog_changed'", + vec![], + ) + .await + .expect("unsupported source"); + commit(transaction).await.expect("source fault commit"); + let unsupported = logical_state(store).await; + assert!( + rebuild_catalog_projection(store) + .await + .expect_err("unsupported schema") + .to_string() + .contains("unsupported catalog fact schema") + ); + assert_eq!(logical_state(store).await, unsupported); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("restore source"); + let (transaction, _) = query( + transaction, + "UPDATE recall_events SET event_schema_version=1 WHERE event_type='warehouse_catalog_changed'", + vec![], + ) + .await + .expect("restore schema"); + commit(transaction).await.expect("restored source commit"); + rebuild_catalog_projection(store) + .await + .expect("retry restored source"); + assert_eq!(logical_state(store).await, expected); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs new file mode 100644 index 0000000000..b1cb6bd08a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs @@ -0,0 +1,54 @@ +//! Full-state acceptance keeps protected live ownership and source facts non-vacuous. + +use crate::runs::_helpers::lifecycle::rebuild::rebuild_run_projections; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; + +pub(crate) async fn run_replay_contract(store: &ControlStore) { + let expected = logical_state(store).await; + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("damage cut"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/projection_replay.json"); + let fixture: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("projection replay fixture")) + .expect("projection replay fixture"); + for sql in fixture["run_damage"].as_array().expect("run damage") { + (transaction, _) = query(transaction, sql.as_str().expect("damage SQL"), vec![]) + .await + .expect("corrupt derived state"); + } + commit(transaction).await.expect("damage commit"); + let damaged = logical_state(store).await; + assert_ne!(damaged, expected); + let fault = TransactionFault { + description: "node repair fails after resetting run metadata", + table: "sqb_run_nodes", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected run replay failure')", + postgres_body: "RAISE EXCEPTION 'injected run replay failure'", + }; + install_fault(store, &fault).await; + let error = rebuild_run_projections(store) + .await + .expect_err("injected failure"); + assert!( + error.to_string().contains("injected run replay failure"), + "{error}" + ); + assert_eq!(logical_state(store).await, damaged); + remove_fault(store, &fault).await; + rebuild_run_projections(store) + .await + .expect("repair runs and nodes"); + assert_eq!(logical_state(store).await, expected); + rebuild_run_projections(store) + .await + .expect("idempotent replay"); + assert_eq!(logical_state(store).await, expected); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs index 0fe7151327..8b6629f0ca 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs @@ -253,6 +253,7 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi BTreeMap::from([("success".into(), 1)]) ); assert_eq!(runs[0].selection_json, claim.request.selection); + crate::runs::tests::replay_helpers::run_replay_contract(store).await; close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index ddc82ea434..46600571e0 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -4,6 +4,8 @@ mod claim_helpers; pub(crate) mod helpers; #[path = "monitor/helpers.rs"] mod monitor_helpers; +#[path = "replay/helpers.rs"] +pub(crate) mod replay_helpers; #[path = "termination/helpers.rs"] pub(crate) mod terminal_helpers; #[path = "test_checks.rs"] @@ -24,6 +26,8 @@ mod test_lifecycle; mod test_monitor; #[path = "test_permissions.rs"] mod test_permissions; +#[path = "test_replay.rs"] +mod test_replay; #[path = "test_requests.rs"] mod test_requests; #[path = "test_reruns.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index 10f1b53bf8..7f74078da3 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -416,6 +416,7 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups runs.iter().all(|run| run.status == RunStatus::Success), "{runs:?}" ); + crate::runs::tests::replay_helpers::run_replay_contract(store).await; completed.push(test_case.description); close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 21ec54e845..030223bace 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -121,6 +121,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .status, RunStatus::Running ); + crate::runs::tests::replay_helpers::run_replay_contract(store).await; let launch_failed = RunChangeRequest { event_id: Uuid::now_v7(), change: RunChange::LaunchFailed { @@ -560,6 +561,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ .expect("claim following run") .expect("released capacity"); assert_eq!(following.request.run_id, Uuid::from_u128(2)); + crate::runs::tests::replay_helpers::run_replay_contract(store).await; close_test_store(fixture).await; }); } @@ -719,6 +721,7 @@ fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_l events[2]["payload"]["change"]["change"]["actor"], "test-operator" ); + crate::runs::tests::replay_helpers::run_replay_contract(store).await; close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_replay.rs b/crates/sqlbuild-orchestration/tests/runs/test_replay.rs new file mode 100644 index 0000000000..a652652ae9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_replay.rs @@ -0,0 +1,193 @@ +//! Causal replay crosses pages and restores a missing projection without rebuilding sources. + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::_helpers::lifecycle::change_run::change_run_in; +use crate::runs::_helpers::lifecycle::rebuild::rebuild_run_projections; +use crate::runs::_helpers::requests::publication::publish_request; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunStatus}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::replay_helpers::run_replay_contract; +use crate::runs::tests::test_types::RunReplayTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use uuid::Uuid; + +#[test] +fn given_paged_run_facts_when_replaying_then_missing_rows_and_original_times_return() { + let test_cases = [ + RunReplayTestCase { + description: "SQLite paged replay", + backend: Backend::Sqlite, + expected_runs: 56, + }, + RunReplayTestCase { + description: "PostgreSQL paged replay", + backend: Backend::Postgres, + expected_runs: 56, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + for id in (1..=55).rev() { + queue( + store, + &request( + (location, published.snapshot_id), + id, + id as i64, + &["customers", "orders"], + ), + ) + .await; + } + let queued_at = list_runs(store, (location, "development"), 100) + .await + .expect("original times") + .into_iter() + .find(|run| run.run_id == Uuid::from_u128(1)) + .expect("original run") + .queued_at; + let replica = second_store(store).await; + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("held writer"); + transaction.now = published.published_at - chrono::Duration::days(1); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let (transaction, applied) = change_run_in( + transaction, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: "clock rollback".into(), + }, + }, + ) + .await + .expect("cancel after a backward clock step"); + assert!(applied); + let (transaction, applied) = publish_request( + transaction, + &request((location, published.snapshot_id), 56, 56, &["orders"]), + ) + .await + .expect("new uncommitted request"); + assert!(applied); + let repair_store = replica.clone(); + let mut repair = + tokio::spawn(async move { rebuild_run_projections(&repair_store).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut repair) + .await + .is_err() + ); + assert_eq!( + list_runs(&replica, (location, "development"), 100) + .await + .expect("old source cut") + .len(), + 55 + ); + commit(transaction).await.expect("publish new source cut"); + tokio::time::timeout(std::time::Duration::from_secs(20), repair) + .await + .expect("repair deadline") + .expect("join repair") + .expect("serialized replay"); + run_replay_contract(store).await; + let expected = logical_state(store).await; + let transaction = begin(store, TransactionMode::Write) + .await + .expect("missing projections"); + let (transaction, _) = query(transaction, "DELETE FROM sqb_run_nodes", vec![]) + .await + .expect("remove nodes"); + let (transaction, _) = query(transaction, "DELETE FROM sqb_runs", vec![]) + .await + .expect("remove metadata"); + commit(transaction) + .await + .expect("missing projections commit"); + rebuild_run_projections(store) + .await + .expect("restore missing runs"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert_eq!(logical_state(store).await, expected); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("replayed states"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == Uuid::from_u128(1)) + .expect("cancelled run") + .status, + RunStatus::Cancelled + ); + let cancelled = runs + .iter() + .find(|run| run.run_id == Uuid::from_u128(1)) + .expect("original identity"); + assert_eq!(cancelled.queued_at, queued_at); + assert_eq!( + cancelled.ended_at, + Some(published.published_at - chrono::Duration::days(1)) + ); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("source fault"); + let (transaction, _) = query( + transaction, + "UPDATE recall_events SET event_schema_version=999 WHERE storage_id=(SELECT MAX(storage_id) FROM recall_events WHERE producer='scheduler')", + vec![], + ) + .await + .expect("unsupported schema"); + commit(transaction).await.expect("source fault commit"); + let unsupported = logical_state(store).await; + assert!( + rebuild_run_projections(store) + .await + .expect_err("unsupported source") + .to_string() + .contains("unsupported run fact schema") + ); + assert_eq!(logical_state(store).await, unsupported); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index cc08da0422..587b55d773 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -338,6 +338,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ "successful work has nothing to retry from failure" ); } + crate::runs::tests::replay_helpers::run_replay_contract(store).await; close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 078f2c0a62..d22a01924f 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -108,3 +108,9 @@ pub(crate) struct MixedAdmissionTestCase { pub(crate) parent_priority: i64, pub(crate) expected_selections: [&'static str; 2], } + +pub(crate) struct RunReplayTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index bf728d526e..f700fcde8d 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -20,7 +20,7 @@ use crate::store::tests::store_helpers::{ }; use crate::store::tests::store_test_types::Backend; -fn snapshot(location_id: Uuid) -> SnapshotInput { +pub(super) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { location_id, target: "development".into(), @@ -325,3 +325,26 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { } recovered.generation } + +pub(super) async fn damage(store: &ControlStore) { + use crate::store::main::{begin::begin, commit::commit, query::query}; + use crate::store::models::TransactionMode; + + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("damage cut"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/projection_replay.json"); + let fixture: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("projection replay fixture")) + .expect("projection replay fixture"); + for sql in fixture["snapshot_damage"] + .as_array() + .expect("snapshot damage") + { + (transaction, _) = query(transaction, sql.as_str().expect("damage SQL"), vec![]) + .await + .expect("projection damage"); + } + commit(transaction).await.expect("damage commits"); +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs index 6963cfddf6..d8b7b352f7 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -4,5 +4,9 @@ mod contract; #[path = "helpers.rs"] mod helpers; +#[path = "test_replay.rs"] +mod replay; +#[path = "test_replay_writers.rs"] +mod replay_writers; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs new file mode 100644 index 0000000000..d45abb7b61 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs @@ -0,0 +1,182 @@ +//! Current pointers and jobs recover from retained facts, independently of trigger replay. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::_helpers::rebuild::rebuild_snapshot_projections; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::tests::helpers::{damage, snapshot}; +use crate::snapshots::tests::test_types::SnapshotTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_corrupt_snapshot_projections_when_replayed_then_full_state_is_restored() { + let test_cases = [ + SnapshotTestCase { + description: "SQLite replay", + backend: Backend::Sqlite, + expected_generation: 7, + }, + SnapshotTestCase { + description: "PostgreSQL replay", + backend: Backend::Postgres, + expected_generation: 7, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "customers":{"description":"Customers","selection":["customers"]}, + "orders":{"description":"Orders","selection":["orders"],"settings":{"threads":3}} + })); + input.documents.insert( + "groups".into(), + json!({ + "nightly":{"description":"Nightly","selection":["customers","orders"], + "jobs":{"customers":[],"orders":["customers"]}} + }), + ); + input.documents.insert("triggers".into(), json!({ + "daily":{"description":"Daily","job_key":"orders","starts_at":"2020-01-01T00:00:00Z", + "schedule":{"kind":"calendar","times":["2020-01-01T00:00:00Z","2030-01-01T00:00:00Z"],"refresh_at":"2029-01-01T00:00:00Z"}} + })); + publish_snapshot(store, &input) + .await + .expect("first generation"); + input.generation = 7; + input.documents.get_mut("jobs").expect("jobs")["orders"]["description"] = + json!("Current orders"); + let latest = publish_snapshot(store, &input) + .await + .expect("current generation"); + assert_eq!( + latest.generation, test_case.expected_generation, + "{}", + test_case.description + ); + let trigger = crate::declarations::main::list_triggers::list_triggers( + store, + location, + "development", + ) + .await + .expect("trigger") + .remove(0); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("lease") + .expect("leader"); + evaluate_trigger(store, &lease, trigger.trigger_id) + .await + .expect("retained run/tick facts"); + input.generation = 1; + for index in 0..101 { + input.target = format!("target_{index:03}"); + publish_snapshot(store, &input) + .await + .expect("independent target"); + } + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + input.location_id = other; + input.target = "development".into(); + input.documents.clear(); + publish_snapshot(store, &input) + .await + .expect("jobless scope"); + let expected = logical_state(store).await; + damage(store).await; + let damaged = logical_state(store).await; + assert_ne!(damaged, expected); + let fault = TransactionFault { + description: "late pointer failure after job repair", + table: "sqb_current_snapshots", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected replay failure')", + postgres_body: "RAISE EXCEPTION 'injected replay failure'", + }; + install_fault(store, &fault).await; + assert!( + rebuild_snapshot_projections(store) + .await + .expect_err("injected failure") + .to_string() + .contains("injected replay failure") + ); + assert_eq!(logical_state(store).await, damaged); + remove_fault(store, &fault).await; + rebuild_snapshot_projections(store).await.expect("replay"); + assert_eq!(logical_state(store).await, expected); + assert_eq!( + current_snapshot(store, location, "development") + .await + .expect("read") + .expect("restored") + .record, + latest + ); + rebuild_snapshot_projections(store) + .await + .expect("idempotent replay"); + assert_eq!(logical_state(store).await, expected); + + damage(store).await; + let transaction = begin(store, TransactionMode::Write) + .await + .expect("source fault"); + let (transaction, rows) = query(transaction, "SELECT document_hash,body FROM sqb_documents WHERE document_hash=(SELECT document_hash FROM sqb_snapshot_documents WHERE snapshot_id=$1 AND kind='jobs')", vec![Parameter::Id(latest.snapshot_id)]).await.expect("source body"); + let body: Vec = serde_json::from_value(rows[0]["body"].clone()).expect("bytes"); + let hash = rows[0]["document_hash"].as_str().expect("hash").to_owned(); + let (transaction, _) = query( + transaction, + "UPDATE sqb_documents SET body=$1 WHERE document_hash=$2", + vec![ + Parameter::Bytes(b"invalid compressed source".to_vec()), + Parameter::Text(hash.clone()), + ], + ) + .await + .expect("malformed retained source"); + commit(transaction).await.expect("source fault commits"); + let malformed = logical_state(store).await; + assert!(rebuild_snapshot_projections(store).await.is_err()); + assert_eq!(logical_state(store).await, malformed); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("restore source"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_documents SET body=$1 WHERE document_hash=$2", + vec![Parameter::Bytes(body), Parameter::Text(hash)], + ) + .await + .expect("source restored"); + commit(transaction).await.expect("commit restored source"); + rebuild_snapshot_projections(store) + .await + .expect("retry restored source"); + assert_eq!(logical_state(store).await, expected); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs new file mode 100644 index 0000000000..08fa4ca5be --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs @@ -0,0 +1,129 @@ +//! Replay and normal publication share the writer fence while readers keep a consistent cut. + +use crate::credentials::tests::location_helpers::create_location; +use crate::snapshots::_helpers::rebuild::rebuild_snapshot_projections; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::tests::helpers::snapshot; +use crate::snapshots::tests::test_types::SnapshotTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_concurrent_publication_when_replaying_then_readers_and_writers_keep_one_generation() { + let test_cases = [ + SnapshotTestCase { + description: "SQLite concurrent replay", + backend: Backend::Sqlite, + expected_generation: 2, + }, + SnapshotTestCase { + description: "PostgreSQL concurrent replay", + backend: Backend::Postgres, + expected_generation: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + serde_json::json!({ + "orders":{"description":"Initial","selection":["orders"]} + }), + ); + let initial = publish_snapshot(store, &input).await.expect("initial"); + let replica = second_store(store).await; + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("held writer"); + for sql in [ + "DELETE FROM sqb_job_nodes", + "DELETE FROM sqb_jobs", + "DELETE FROM sqb_current_snapshots", + ] { + (transaction, _) = query(transaction, sql, vec![]) + .await + .expect("uncommitted damage"); + } + let (ready, started) = tokio::sync::oneshot::channel(); + let repair_store = replica.clone(); + let mut repair = tokio::spawn(async move { + ready.send(()).expect("repair ready"); + rebuild_snapshot_projections(&repair_store).await + }); + started.await.expect("repair started"); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut repair) + .await + .is_err() + ); + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["orders"]["description"] = + serde_json::json!("Current"); + input.documents.get_mut("jobs").expect("jobs")["orders"]["selection"] = + serde_json::json!(["customers", "orders"]); + let (ready, started) = tokio::sync::oneshot::channel(); + let publisher_store = replica.clone(); + let publisher = tokio::spawn(async move { + ready.send(()).expect("publisher ready"); + publish_snapshot(&publisher_store, &input).await + }); + started.await.expect("publisher started"); + assert!(!repair.is_finished()); + assert!(!publisher.is_finished()); + assert_eq!( + current_snapshot(&replica, location, "development") + .await + .expect("concurrent reader") + .expect("previous generation") + .record, + initial + ); + commit(transaction).await.expect("release writer"); + tokio::time::timeout(std::time::Duration::from_secs(20), repair) + .await + .expect("repair deadline") + .expect("join repair") + .expect("repair"); + let published = tokio::time::timeout(std::time::Duration::from_secs(20), publisher) + .await + .expect("publisher deadline") + .expect("join publisher") + .expect("publish"); + let current = current_snapshot(&replica, location, "development") + .await + .expect("final reader") + .expect("current generation"); + assert_eq!(current.record, published); + assert_eq!( + current.record.generation, test_case.expected_generation, + "{}", + test_case.description + ); + assert_eq!( + current.documents["jobs"]["orders"]["description"], + "Current" + ); + let final_state = logical_state(&replica).await; + rebuild_snapshot_projections(store) + .await + .expect("replay newest cut"); + assert_eq!(logical_state(&replica).await, final_state); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs b/crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs new file mode 100644 index 0000000000..e13e1321dc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs @@ -0,0 +1,42 @@ +//! Hash complete logical state without printing credential or evidence rows on a failure. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) async fn logical_state(store: &ControlStore) -> BTreeMap { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("state cut"); + let sql = [ + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name", + "SELECT table_name AS name FROM information_schema.tables WHERE table_schema=current_schema() AND table_type='BASE TABLE' ORDER BY table_name", + ][usize::from(matches!(store, ControlStore::Postgres(..)))]; + let (mut transaction, tables) = query(transaction, sql, vec![]).await.expect("tables"); + let mut result = BTreeMap::new(); + for table in tables { + let name = table["name"].as_str().expect("table name"); + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT * FROM \"{}\"", name.replace('"', "\"\"")), + vec![], + ) + .await + .expect("logical rows"); + let mut rows: Vec = rows + .iter() + .map(|row| serde_json::to_string(row).expect("canonical row")) + .collect(); + rows.sort(); + result.insert( + name.to_owned(), + sha256_hex(&serde_json::to_vec(&rows).expect("rows")), + ); + } + commit(transaction).await.expect("finish state cut"); + result +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs index a257c9aeb9..868430a934 100644 --- a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -8,6 +8,8 @@ pub(crate) mod fault_types; pub(crate) mod helpers; #[path = "leases/helpers.rs"] pub(crate) mod lease_helpers; +#[path = "logical_state/helpers.rs"] +pub(crate) mod logical_state; #[path = "replicas/helpers.rs"] pub(crate) mod replica_helpers; #[path = "helpers.rs"] From 3b49c5359b36084123710fc8eb080c42fd899d24 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:54:14 +0000 Subject: [PATCH 116/138] test: verify recovery across repeated supervisor crashes --- .../local_recovery_publication_fault.sql | 5 ++ .../publication/test_local_recovery.py | 90 +++++++++++++++---- 2 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql new file mode 100644 index 0000000000..a7ddaa159f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql @@ -0,0 +1,5 @@ +-- Crash successive supervisors while a stopped child's terminal publication is rejected. +-- Claims and the inherited ownership fence must survive until the terminal fact commits. +CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs +WHEN NEW.status = 'failed' +BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END; diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index 938a1b34b3..a9533cdef3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -104,17 +104,15 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an "test_case", [ WorkerExecutionTestCase( - "pre-handshake child survives restart and failed terminal publication retains its claim", + "successive supervisor crashes retain a stopped child's claim until publication commits", "SELECT 23 AS amount", "", "failed", [], launcher="uncooperative", - store_setup=( - "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " - "WHEN NEW.status='failed' " - "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" - ), + store_setup=Path( + "benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql" + ).read_text(), ) ], ids=lambda case: case.description, @@ -128,20 +126,32 @@ def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_w worker.api, worker.process, worker.log, pending=frozenset({"queued"}) ) assert active["status"] == "launched" + tables: tuple[str, ...] = ("sqb_write_claims", "sqb_concurrency_slots", "sqb_run_liveness") + with sqlite3.connect(worker.control_store) as control: + retained: dict[str, list[tuple[Any, ...]]] = { + table: control.execute(f"SELECT * FROM {table}").fetchall() for table in tables + } + request: tuple[Any, ...] = control.execute( + "SELECT request_json FROM sqb_runs" + ).fetchone() + claim_id: str = control.execute( + "SELECT worker_instance FROM sqb_run_liveness" + ).fetchone()[0] + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ] handle: int = os.pidfd_open(pid) try: worker.process.kill() assert worker.process.wait(timeout=10) == -signal.SIGKILL log: Path = tmp_path / "replacement-scheduler.log" with location_process( - [ - str(Path(sys.executable).with_name("sqb")), - "scheduler", - "--database-env", - "SQLBUILD_SCHEDULER_TEST_STORE", - "--launchers", - str(tmp_path / "launchers.json"), - ], + arguments, log, tmp_path / "replacement-scheduler.stdout", ) as process: @@ -157,12 +167,48 @@ def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_w signal.pidfd_send_signal(handle, signal.SIGKILL) wait_for_log(process, log, "Retrying worker shutdown publication") assert process.poll() is None, log.read_text() + assert "Worker launched for run" not in log.read_text() with sqlite3.connect(worker.control_store) as control: assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( 1, ) - _ = control.execute("DROP TRIGGER reject_terminal") - assert process.wait(timeout=10) != 0, log.read_text() + assert { + table: control.execute(f"SELECT * FROM {table}").fetchall() + for table in tables + } == retained + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (0,) + process.kill() + assert process.wait(timeout=10) == -signal.SIGKILL + log = tmp_path / "second-replacement-scheduler.log" + with location_process( + arguments, log, tmp_path / "second-replacement-scheduler.stdout" + ) as process: + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + wait_for_log(process, log, "Retrying worker shutdown publication") + assert process.poll() is None, log.read_text() + assert "Worker launched for run" not in log.read_text() + assert int((worker.project / "worker.pid").read_text()) == pid + with sqlite3.connect(worker.control_store) as control: + assert { + table: control.execute(f"SELECT * FROM {table}").fetchall() + for table in tables + } == retained + assert ( + control.execute("SELECT request_json FROM sqb_runs").fetchone() == request + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (0,) + lock: Path = worker.project / ".sqlbuild/workers" / f"{claim_id}.lock" + with lock.open("rb") as ownership: + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + with sqlite3.connect(worker.control_store) as control: + _ = control.execute("DROP TRIGGER reject_terminal") + assert process.wait(timeout=10) != 0, log.read_text() + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) with sqlite3.connect(worker.control_store) as control: assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( test_case.expected_status, @@ -170,6 +216,18 @@ def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_w assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( 0, ) + assert control.execute( + "SELECT count(*) FROM sqb_concurrency_slots" + ).fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == ( + 0, + ) + assert ( + control.execute("SELECT request_json FROM sqb_runs").fetchone() == request + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) assert control.execute( "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" ).fetchone() == (1,) From 13a895dfc9f8eb669562b429f16c85cad65a6187 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:54:13 +0000 Subject: [PATCH 117/138] feat: rebuild control projections from durable semantic facts --- .../orchestration/projection_replay.json | 5 + .../sqlbuild-cli/src/argv/command_table.json | 8 + .../argv/tests/test_parse_cli_arguments.rs | 11 + .../src/native_commands/_helpers/execution.rs | 31 ++- .../main/run_control_invocation.rs | 31 ++- .../declarations/_helpers/operator_state.rs | 10 +- .../src/declarations/_helpers/publication.rs | 18 ++ .../src/declarations/main/mod.rs | 2 + .../src/declarations/main/rebuild_triggers.rs | 154 +++++++++++ .../main/record_trigger_progress.rs | 40 +++ .../src/declarations/models.rs | 23 ++ .../src/platform/main/mod.rs | 1 + .../src/platform/main/rebuild_projections.rs | 19 ++ .../src/runs/_helpers/lifecycle/mod.rs | 1 - .../src/runs/_helpers/requests/job.rs | 2 +- .../src/runs/main/frozen_job_request.rs | 2 +- .../src/runs/main/mod.rs | 2 +- .../rebuild.rs => main/replay_projections.rs} | 18 +- .../src/scheduler/_helpers/persistence.rs | 15 +- .../src/sensors/_helpers/attempts.rs | 12 +- .../src/snapshots/_helpers/mod.rs | 1 - .../{runs => snapshots}/main/job_write_set.rs | 2 +- .../src/snapshots/main/mod.rs | 2 + .../rebuild_projections.rs} | 18 +- .../src/store/_helpers/migration.rs | 44 ++++ .../src/store/main/migrate_store.rs | 50 +--- .../src/store/main/mod.rs | 1 + .../src/store/main/verify_store.rs | 26 ++ .../src/store/trigger_progress.sql | 8 + .../warehouse_metadata/_helpers/catalog.rs | 65 +---- .../src/warehouse_metadata/main/mod.rs | 1 + .../main/rebuild_catalog.rs | 56 ++++ .../tests/runs/replay/helpers.rs | 8 +- .../tests/runs/test_replay.rs | 3 +- .../tests/scheduler/ticks/test_progress.rs | 246 ++++++++++++++++++ .../tests/scheduler/ticks/test_ticks.rs | 2 + .../tests/sensors/test_recovery.rs | 21 +- .../tests/snapshots/helpers.rs | 10 +- .../tests/snapshots/test_replay.rs | 3 +- .../tests/snapshots/test_replay_writers.rs | 8 +- .../tests/store/helpers.rs | 2 + .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/server.md | 61 +++++ .../publication/_test_types.py | 13 + .../publication/replay/helpers.py | 63 +++++ .../publication/test_projection_rebuild.py | 140 ++++++++++ .../publication/wheel/helpers.py | 13 + website/src/content/docs/docs/cli/server.mdx | 60 +++++ website/src/sidebar.json | 1 + 49 files changed, 1170 insertions(+), 164 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/rebuild_triggers.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/record_trigger_progress.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/rebuild_projections.rs rename crates/sqlbuild-orchestration/src/runs/{_helpers/lifecycle/rebuild.rs => main/replay_projections.rs} (67%) rename crates/sqlbuild-orchestration/src/{runs => snapshots}/main/job_write_set.rs (91%) rename crates/sqlbuild-orchestration/src/snapshots/{_helpers/rebuild.rs => main/rebuild_projections.rs} (79%) create mode 100644 crates/sqlbuild-orchestration/src/store/main/verify_store.rs create mode 100644 crates/sqlbuild-orchestration/src/store/trigger_progress.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/rebuild_catalog.rs create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/replay/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_projection_rebuild.py create mode 100644 website/src/content/docs/docs/cli/server.mdx diff --git a/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json b/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json index f782b7aa99..97b54eb3d9 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json +++ b/benchmarks/duckshop/edge_cases/orchestration/projection_replay.json @@ -9,5 +9,10 @@ "run_damage": [ "DELETE FROM sqb_run_nodes", "UPDATE sqb_runs SET status='queued',node_counts_json='{}',request_json='{}',priority=-99,status_reason='damaged',started_at=NULL,ended_at=NULL,cancel_requested_at=NULL,last_event_storage_id=0" + ], + "trigger_damage": [ + "UPDATE sqb_triggers SET name=name || '_damaged',definition_json='{}'", + "UPDATE sqb_trigger_state SET last_tick_for=NULL,next_due_at=NULL,status='paused'", + "DELETE FROM sqb_current_snapshots" ] } diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index c17fbc2ced..dff1f9d3c2 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1036,6 +1036,14 @@ "help": "show this help message and exit", "group": 1 }, + { + "kind": "store", + "dest": "server_action", + "nargs": "?", + "choices": ["rebuild-projections"], + "help": "repair derived control projections atomically from retained facts in an initialized store", + "group": 0 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index 2524c91185..4d8c3a71ee 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -55,6 +55,17 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "control_default_role", expected_value: Value::Str("none".into()), }, + ParsedValueTestCase { + description: "projection repair selects maintenance instead of starting server roles", + argv: &[ + "server", + "rebuild-projections", + "--database-env", + "SQLBUILD_TEST_CONTROL_DATABASE_URL", + ], + attribute: "server_action", + expected_value: Value::Str("rebuild-projections".into()), + }, ParsedValueTestCase { description: "viewer metadata access requires an explicit server option", argv: &["server", "--default-role", "viewer"], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 01e48afdb5..3e1c4ca854 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -15,6 +15,7 @@ use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, D use sqlbuild_orchestration::authorization::main::recover_administrator::recover_administrator; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::platform::main::rebuild_projections::rebuild_projections; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; use sqlbuild_orchestration::platform::models::{ @@ -64,11 +65,13 @@ use crate::native_commands::_helpers::terminal::{ finish_lost_stdout, terminal_columns, write_output, }; use crate::native_commands::constants::{ - COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, METADATA_RECONCILE_SECONDS, - UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, + METADATA_RECONCILE_SECONDS, SCHEDULER_COMMAND, SERVER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, + UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; +use crate::native_commands::main::_run_control::run_control; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, @@ -871,6 +874,30 @@ pub(crate) fn start_platform_roles( Ok(()) } +/// Dispatch maintenance actions without implicitly initializing or upgrading their store. +pub(crate) async fn run_control_action( + namespace: &Namespace, + command: &str, +) -> Result, ControlError> { + if command == ADMIN_COMMAND { + return run_administration(namespace).await.map(Some); + } + if command == SERVER_COMMAND + && text(namespace, "server_action").as_deref() == Some("rebuild-projections") + { + drop(write_output(&[( + Stream::Stderr, + "Rebuilding control projections...\n", + )])); + let store = open_store(&control_database(namespace)?).await?; + rebuild_projections(&store).await?; + return Ok(Some(json!({"rebuilt":true}))); + } + run_control(namespace, command == SCHEDULER_COMMAND) + .await + .map(|()| None) +} + /// Keep exclusive offline recovery outside the online store-opening and migration path. pub(crate) async fn run_administration( namespace: &Namespace, diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs index d56e98354b..a49f6439f4 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -4,10 +4,9 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{flag, text}; -use crate::native_commands::_helpers::execution::run_administration; +use crate::native_commands::_helpers::execution::run_control_action; use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; -use crate::native_commands::main::_run_control::run_control; pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { let command = text(namespace, "command")?; @@ -47,24 +46,26 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { } }; let admin = command == ADMIN_COMMAND; - let role = if admin { "administration" } else { "service" }; - let result = runtime.block_on(async { - if admin { - run_administration(namespace).await.map(Some) - } else { - run_control(namespace, command == SCHEDULER_COMMAND) - .await - .map(|()| None) - } - }); + let rebuild = command == SERVER_COMMAND + && text(namespace, "server_action").as_deref() == Some("rebuild-projections"); + let role = if admin { + "administration" + } else if rebuild { + "projection rebuild" + } else { + "service" + }; + let result = runtime.block_on(run_control_action(namespace, &command)); Some(match result { Ok(output) => { let lost = output.and_then(|value| write_output(&[(Stream::Stdout, &format!("{value}\n"))])); - if admin && lost.is_some() { + if (admin || rebuild) && lost.is_some() { drop(write_output(&[( Stream::Stderr, - "Control administration JSON result delivery failed; changes may already be committed\n", + &format!( + "Control {role} JSON result delivery failed; changes may already be committed\n" + ), )])); return Some(1); } @@ -72,6 +73,8 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { if exit_code == 0 { let message = if admin { "Control administration completed successfully\n" + } else if rebuild { + "Control projections rebuilt successfully\n" } else { "Control service stopped successfully\n" }; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs index 0bae32b69c..21e9f69509 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs @@ -3,6 +3,7 @@ use serde_json::{Value, json}; use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::main::record_trigger_progress::record_trigger_progress; use crate::declarations::models::{ EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerDeclaration, TriggerStatus, }; @@ -73,10 +74,15 @@ pub(crate) async fn apply_operator_change( }); } let now = transaction.now; - let (mut transaction, _) = query(transaction, - "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", + let (mut transaction, changed) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6 \ + AND (status IS DISTINCT FROM $1 OR status_override IS DISTINCT FROM $2 OR last_tick_for IS DISTINCT FROM $3 \ + OR next_due_at IS DISTINCT FROM $4) RETURNING trigger_id", vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + if !changed.is_empty() { + transaction = record_trigger_progress(transaction, request.trigger_id).await?; + } transaction .changes .insert(Change::Trigger(request.trigger_id)); diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs index 45f8ad5de8..e346c93062 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs @@ -3,6 +3,7 @@ use uuid::Uuid; use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; +use crate::declarations::main::record_trigger_progress::record_trigger_progress; use crate::declarations::models::{Definitions, PreviousTriggerState}; use crate::sensors::main::publish_sensors::publish_sensors; use crate::snapshots::models::SnapshotRecord; @@ -59,6 +60,23 @@ pub(crate) async fn publish_projections( ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; } + let mut cursor = None; + loop { + let rows; + (transaction, rows) = query(transaction, + "SELECT trigger_id FROM sqb_triggers WHERE location_id=$1 AND target=$2 \ + AND ($3 IS NULL OR trigger_id>$3) AND (snapshot_id=$4 OR removed_in_snapshot_id=$4) ORDER BY trigger_id LIMIT 100", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::OptionalId(cursor), Parameter::Id(snapshot.snapshot_id)]).await?; + if rows.is_empty() { + break; + } + for row in rows { + let id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?; + transaction = record_trigger_progress(transaction, id).await?; + cursor = Some(id); + } + } Ok(transaction) } diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index ec650733ce..a5a91cda20 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -7,4 +7,6 @@ pub mod list_triggers; pub mod pause_development_triggers; pub(crate) mod publish_definitions; pub(crate) mod rebuild_jobs; +pub(crate) mod rebuild_triggers; +pub(crate) mod record_trigger_progress; pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/rebuild_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/rebuild_triggers.rs new file mode 100644 index 0000000000..426a6ffc70 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/rebuild_triggers.rs @@ -0,0 +1,154 @@ +//! Restore recorded trigger images; a missing legacy source aborts the caller's entire repair. + +use std::collections::BTreeSet; + +use sqlbuild_recall::schema::models::ColumnType; +use uuid::Uuid; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::models::{TriggerDeclaration, TriggerProjection}; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +async fn expected_triggers( + mut transaction: ControlTransaction, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let mut expected: BTreeSet = BTreeSet::new(); + let mut cursor = None; + loop { + let rows; + (transaction, rows) = query(transaction, + "SELECT * FROM sqb_snapshots WHERE ($1 IS NULL OR snapshot_id>$1) ORDER BY snapshot_id LIMIT 100", + vec![Parameter::OptionalId(cursor)]).await?; + if rows.is_empty() { + break; + } + for row in rows { + if row["snapshot_schema_version"].as_i64() != Some(1) { + return Err(ControlError::Invalid( + "cannot replay an unsupported snapshot schema".into(), + )); + } + let snapshot: SnapshotRecord = decode_row(row)?; + let documents; + (transaction, documents) = + read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["triggers"])) + .await?; + let definitions = decode_definitions(&documents)?; + for (kind, name) in definitions + .triggers + .keys() + .map(|name| ("schedule", name)) + .chain(definitions.sensors.keys().map(|name| ("sensor", name))) + { + expected.insert(Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, kind, name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + )); + } + cursor = Some(snapshot.snapshot_id); + } + } + Ok((transaction, expected)) +} + +pub(crate) async fn rebuild_triggers( + transaction: ControlTransaction, +) -> Result { + let (mut transaction, mut expected) = expected_triggers(transaction).await?; + let mut cursor = None; + loop { + let rows; + (transaction, rows) = query(transaction, + "SELECT p.* FROM sqb_trigger_progress p WHERE ($1 IS NULL OR p.trigger_id>$1) \ + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_progress newer WHERE newer.trigger_id=p.trigger_id AND newer.storage_id>p.storage_id) \ + ORDER BY p.trigger_id LIMIT 100", vec![Parameter::OptionalId(cursor)]).await?; + if rows.is_empty() { + break; + } + for row in rows { + if row["schema_version"].as_i64() != Some(1) { + return Err(ControlError::Invalid( + "cannot replay an unsupported trigger progress schema".into(), + )); + } + let projection: TriggerProjection = + serde_json::from_value(row["projection_json"].clone()).map_err(|error| { + ControlError::Invalid(format!("invalid trigger progress: {error}")) + })?; + let identity = Uuid::new_v5( + &projection.location_id, + &serde_json::to_vec(&(&projection.target, &projection.kind, &projection.name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + if !expected.remove(&identity) + || identity != projection.trigger_id + || row["trigger_id"] != serde_json::json!(identity) + || row["snapshot_id"] != serde_json::json!(projection.snapshot_id) + || !matches!( + (&*projection.kind, &projection.definition_json), + ("schedule", TriggerDeclaration::Schedule(_)) + | ("sensor", TriggerDeclaration::Sensor(_)) + ) + { + return Err(ControlError::Invalid( + "trigger progress does not match retained declarations".into(), + )); + } + cursor = Some(identity); + transaction = restore_trigger(transaction, projection).await?; + } + } + if !expected.is_empty() { + return Err(ControlError::Invalid( + "cannot rebuild triggers: retained legacy history lacks semantic progress; no cursor was inferred".into(), + )); + } + for table in ["sqb_trigger_state", "sqb_triggers"] { + (transaction, _) = query(transaction, + &format!("DELETE FROM {table} WHERE NOT EXISTS (SELECT 1 FROM sqb_trigger_progress p WHERE p.trigger_id={table}.trigger_id)"), + vec![]).await?; + } + Ok(transaction) +} + +async fn restore_trigger( + transaction: ControlTransaction, + projection: TriggerProjection, +) -> Result { + let definition = serde_json::to_value(&projection.definition_json) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, source) = query(transaction, + "SELECT snapshot_id FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3", + vec![Parameter::Id(projection.snapshot_id), Parameter::Id(projection.location_id), Parameter::Text(projection.target.clone())]).await?; + if source.is_empty() { + return Err(ControlError::Invalid( + "trigger progress has no matching publication".into(), + )); + } + let (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id,location_id,target,kind,name,job_key,definition_json,snapshot_id,removed_in_snapshot_id) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id) DO UPDATE SET location_id=excluded.location_id,target=excluded.target, \ + kind=excluded.kind,name=excluded.name,job_key=excluded.job_key,definition_json=excluded.definition_json, \ + snapshot_id=excluded.snapshot_id,removed_in_snapshot_id=excluded.removed_in_snapshot_id", + vec![Parameter::Id(projection.trigger_id),Parameter::Id(projection.location_id),Parameter::Text(projection.target), + Parameter::Text(projection.kind),Parameter::Text(projection.name),Parameter::Text(projection.job_key),Parameter::Json(definition), + Parameter::Id(projection.snapshot_id),Parameter::OptionalId(projection.removed_in_snapshot_id)]).await?; + let (mut transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id,status,status_override,started_at,cursor_json,last_tick_for,next_due_at,consecutive_failures,updated_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status,status_override=excluded.status_override, \ + started_at=excluded.started_at,cursor_json=excluded.cursor_json,last_tick_for=excluded.last_tick_for,next_due_at=excluded.next_due_at, \ + consecutive_failures=excluded.consecutive_failures,updated_at=excluded.updated_at", + vec![Parameter::Id(projection.trigger_id),Parameter::Text(projection.status.as_str().into()), + Parameter::OptionalText(projection.status_override.map(|value| value.as_str().into())),Parameter::Time(projection.started_at), + projection.cursor_json.map_or(Parameter::Null(ColumnType::Json), Parameter::Json),Parameter::OptionalTime(projection.last_tick_for), + Parameter::OptionalTime(projection.next_due_at),Parameter::Integer(projection.consecutive_failures),Parameter::Time(projection.updated_at)]).await?; + transaction + .changes + .insert(Change::Trigger(projection.trigger_id)); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/record_trigger_progress.rs b/crates/sqlbuild-orchestration/src/declarations/main/record_trigger_progress.rs new file mode 100644 index 0000000000..d4694a9308 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/record_trigger_progress.rs @@ -0,0 +1,40 @@ +//! Record immutable semantic progress in the transition owner's existing transaction. + +use uuid::Uuid; + +use crate::declarations::models::TriggerProjection; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn record_trigger_progress( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result { + let (transaction, rows) = query( + transaction, + "SELECT t.*,s.status,s.status_override,s.started_at,s.cursor_json,s.last_tick_for,\ + s.next_due_at,s.consecutive_failures,s.updated_at FROM sqb_triggers t \ + JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id WHERE t.trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await?; + let projection: TriggerProjection = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Schema("trigger transition has no complete projection".into()) + })?)?; + let body = serde_json::to_value(&projection) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, previous) = query(transaction, + "SELECT schema_version,projection_json FROM sqb_trigger_progress WHERE trigger_id=$1 ORDER BY storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + if previous.first().is_some_and(|row| { + row["schema_version"].as_i64() == Some(1) && row["projection_json"] == body + }) { + return Ok(transaction); + } + let (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_progress (storage_id,trigger_id,snapshot_id,schema_version,projection_json) \ + SELECT COALESCE(MAX(storage_id),0)+1,$1,$2,1,$3 FROM sqb_trigger_progress", + vec![Parameter::Id(trigger_id), Parameter::Id(projection.snapshot_id), Parameter::Json(body)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 5231762051..409d76b88c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -113,6 +113,29 @@ pub struct TriggerRecord { pub next_due_at: Option>, } +/// Recorded only by live transition owners, never seeded from a legacy projection during replay. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TriggerProjection { + pub(crate) trigger_id: uuid::Uuid, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) job_key: String, + pub(crate) definition_json: TriggerDeclaration, + pub(crate) snapshot_id: uuid::Uuid, + pub(crate) removed_in_snapshot_id: Option, + pub(crate) status: TriggerStatus, + pub(crate) status_override: Option, + pub(crate) started_at: DateTime, + pub(crate) cursor_json: Option, + pub(crate) last_tick_for: Option>, + pub(crate) next_due_at: Option>, + pub(crate) consecutive_failures: i64, + pub(crate) updated_at: DateTime, +} + /// Operator changes are immutable facts; clearing an override restores the published code default. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 86c2014360..b817b44415 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -6,6 +6,7 @@ pub mod location_work; pub(crate) mod read_location_work; pub mod read_person_connection_test; pub(crate) mod read_platform_events; +pub mod rebuild_projections; pub mod report_location_health; pub mod report_remote_location_health; pub mod request_location_work; diff --git a/crates/sqlbuild-orchestration/src/platform/main/rebuild_projections.rs b/crates/sqlbuild-orchestration/src/platform/main/rebuild_projections.rs new file mode 100644 index 0000000000..a1d10ca4cf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/rebuild_projections.rs @@ -0,0 +1,19 @@ +//! Repair projections on one serialized cut; no phase commits independently or rewrites facts. + +use crate::declarations::main::rebuild_triggers::rebuild_triggers; +use crate::runs::main::replay_projections::replay_run_projections; +use crate::snapshots::main::rebuild_projections::replay_snapshot_projections; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, verify_store::verify_store}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_metadata::main::rebuild_catalog::replay_catalog_projection; + +/// Direct control-store ownership is required; this is not a remotely granted operation. +pub async fn rebuild_projections(store: &ControlStore) -> Result<(), ControlError> { + let transaction = verify_store(store, begin(store, TransactionMode::Write).await?).await?; + let transaction = replay_snapshot_projections(transaction).await?; + let transaction = rebuild_triggers(transaction).await?; + let transaction = replay_run_projections(transaction).await?; + let transaction = replay_catalog_projection(transaction).await?; + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index c938ed162d..47e55dc5df 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -9,6 +9,5 @@ pub(in crate::runs) mod projection; pub(in crate::runs) mod publish_change; pub(in crate::runs) mod queued; pub(in crate::runs) mod reads; -pub(in crate::runs) mod rebuild; pub(in crate::runs) mod recall; pub(in crate::runs) mod replay; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index a2c223fbb9..1be25f77ee 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -5,8 +5,8 @@ use std::collections::BTreeSet; use uuid::Uuid; use crate::declarations::main::frozen_definitions::frozen_definitions; -use crate::runs::main::job_write_set::job_write_set; use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index 4ca7646a6d..b8fcc623a1 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -2,8 +2,8 @@ use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; -use crate::runs::main::job_write_set::job_write_set; use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index eac7e5ba0e..c7f85ae576 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -4,7 +4,6 @@ pub mod change_run; pub mod claim_next_run; pub(crate) mod frozen_job_request; pub mod heartbeat_run; -pub(crate) mod job_write_set; pub mod list_person_runs; pub mod list_run_claims; pub mod list_run_nodes; @@ -14,6 +13,7 @@ pub mod observe_run; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; +pub(crate) mod replay_projections; pub mod request_job_run; pub mod request_person_job_run; pub mod request_rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs b/crates/sqlbuild-orchestration/src/runs/main/replay_projections.rs similarity index 67% rename from crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs rename to crates/sqlbuild-orchestration/src/runs/main/replay_projections.rs index fbf5829aa5..4423b34874 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/rebuild.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/replay_projections.rs @@ -1,18 +1,14 @@ -//! Reconstruct run metadata and node outcomes on one serialized, atomic source cut. +//! Reconstruct run and node projections within the caller's serialized source cut. use crate::runs::_helpers::lifecycle::replay::replay_run_event; use crate::runs::models::RunReplayRow; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; -/// Internal owner only; liveness, claims, tokens, requests, audits and Recall remain untouched. -#[cfg_attr( - not(test), - expect(dead_code, reason = "awaiting the complete rebuild command") -)] -pub(crate) async fn rebuild_run_projections(store: &ControlStore) -> Result<(), ControlError> { - let transaction = begin(store, TransactionMode::Write).await?; +pub(crate) async fn replay_run_projections( + transaction: ControlTransaction, +) -> Result { let (mut transaction, unsupported) = query(transaction, "SELECT r.run_id FROM sqb_runs r WHERE NOT EXISTS (SELECT 1 FROM recall_events e \ WHERE e.producer='scheduler' AND e.event_type='run_queued' AND e.run_id=CAST(r.run_id AS TEXT) \ @@ -38,5 +34,5 @@ pub(crate) async fn rebuild_run_projections(store: &ControlStore) -> Result<(), transaction = replay_run_event(transaction, row).await?; } } - commit(transaction).await + Ok(transaction) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index cafcbb9a7c..7181da7924 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -2,10 +2,11 @@ use serde_json::json; -use crate::runs::main::job_write_set::job_write_set; +use crate::declarations::main::record_trigger_progress::record_trigger_progress; use crate::runs::main::queue_run::queue_run; use crate::runs::models::{RunKind, RunOrigin, RunRequest}; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; +use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{Change, ControlTransaction, Parameter}; @@ -73,8 +74,16 @@ pub(crate) async fn persist_ticks( transaction = queue_run(transaction, &request).await?; } } - (transaction, _) = query(transaction, - "UPDATE sqb_trigger_state SET last_tick_for=$1, next_due_at=$2, updated_at=$3 WHERE trigger_id=$4", + let changed; + (transaction, changed) = query(transaction, + "UPDATE sqb_trigger_state SET last_tick_for=$1, next_due_at=$2, updated_at=$3 WHERE trigger_id=$4 \ + AND (last_tick_for IS DISTINCT FROM $1 OR next_due_at IS DISTINCT FROM $2) RETURNING trigger_id", vec![Parameter::OptionalTime(plan.last_tick_for), Parameter::OptionalTime(plan.next_due_at), Parameter::Time(now), Parameter::Id(trigger.trigger_id)]).await?; + if !changed.is_empty() { + transaction = record_trigger_progress(transaction, trigger.trigger_id).await?; + transaction + .changes + .insert(Change::Trigger(trigger.trigger_id)); + } Ok(transaction) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs index 310fde7a9b..5f32be2c59 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, Utc}; use uuid::Uuid; +use crate::declarations::main::record_trigger_progress::record_trigger_progress; use crate::sensors::models::{DueSensor, SensorEvaluationFact}; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -51,9 +52,10 @@ pub(crate) async fn defer_sensor( next: DateTime, ) -> Result { let now = transaction.now; - let (transaction, _) = query( + let (transaction, changed) = query( transaction, - "UPDATE sqb_trigger_state SET next_due_at=$1,updated_at=$2 WHERE trigger_id=$3", + "UPDATE sqb_trigger_state SET next_due_at=$1,updated_at=$2 WHERE trigger_id=$3 \ + AND next_due_at IS DISTINCT FROM $1 RETURNING trigger_id", vec![ Parameter::Time(next), Parameter::Time(now), @@ -61,5 +63,9 @@ pub(crate) async fn defer_sensor( ], ) .await?; - Ok(transaction) + if changed.is_empty() { + Ok(transaction) + } else { + record_trigger_progress(transaction, trigger_id).await + } } diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs index 344b06b915..8853fb96b6 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -2,4 +2,3 @@ pub(crate) mod content; pub(crate) mod preparation; pub(crate) mod publication; pub(crate) mod reading; -pub(crate) mod rebuild; diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs similarity index 91% rename from crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs index 91840568ee..3060d1b261 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs @@ -1,4 +1,4 @@ -//! Resolve physical writers from the pinned snapshot and its published job membership. +//! Resolve a job's physical writers from its pinned snapshot. use std::collections::BTreeSet; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index ba10af4391..5f9ca853fc 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,6 +1,7 @@ pub(super) mod _publish_snapshot; pub mod current_snapshot; pub mod current_snapshot_record; +pub(crate) mod job_write_set; pub mod next_remote_generation; pub(crate) mod publish_development_snapshot; pub mod publish_remote_snapshot; @@ -8,4 +9,5 @@ pub mod publish_snapshot; pub(crate) mod read_current_record; pub(crate) mod read_snapshot_documents; pub(crate) mod read_snapshot_nodes; +pub(crate) mod rebuild_projections; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs b/crates/sqlbuild-orchestration/src/snapshots/main/rebuild_projections.rs similarity index 79% rename from crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/rebuild_projections.rs index 7bccf62192..3febfd4724 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/rebuild.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/rebuild_projections.rs @@ -1,18 +1,14 @@ -//! Restore current publication pointers and jobs without republishing or touching trigger state. +//! Replay current publication pointers and jobs without republishing or changing trigger state. use crate::declarations::main::rebuild_jobs::rebuild_jobs; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; -/// Internal replay owner; current pointers and jobs change, never trigger history. -#[cfg_attr( - not(test), - expect(dead_code, reason = "awaiting the complete rebuild command") -)] -pub(crate) async fn rebuild_snapshot_projections(store: &ControlStore) -> Result<(), ControlError> { - let mut transaction = begin(store, TransactionMode::Write).await?; +pub(crate) async fn replay_snapshot_projections( + mut transaction: ControlTransaction, +) -> Result { let mut cursor = None; loop { let (after, parameters) = cursor.as_ref().map_or_else( @@ -58,5 +54,5 @@ pub(crate) async fn rebuild_snapshot_projections(store: &ControlStore) -> Result &format!("DELETE FROM {table} WHERE NOT EXISTS (SELECT 1 FROM sqb_snapshots s WHERE s.location_id={table}.location_id AND s.target={table}.target)"), vec![]).await?; } - commit(transaction).await + Ok(transaction) } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index a270a12be8..66bc5198e7 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -1,7 +1,9 @@ //! One schema template, rendered only at the database type boundary. use semver::Version; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; use crate::store::models::ControlStore; use crate::store::types::Row; @@ -38,6 +40,48 @@ pub(crate) const WAREHOUSE_CATALOG: &str = include_str!("../warehouse_catalog.sq pub(crate) const FINGERPRINT_LOOKUP: &str = include_str!("../fingerprint_lookup.sql"); pub(crate) const WAREHOUSE_STATISTICS: &str = include_str!("../warehouse_statistics.sql"); pub(crate) const CHECK_OBSERVATIONS: &str = include_str!("../check_observations.sql"); +pub(crate) const TRIGGER_PROGRESS: &str = include_str!("../trigger_progress.sql"); + +pub(crate) const ADDITIONS: &[(&str, &str)] = &[ + ("location_instances", LOCATION_INSTANCES), + ("location_work", LOCATION_WORK), + ("platform_sink", PLATFORM_SINK), + ("sensors", SENSORS), + ("run_sensors", RUN_SENSORS), + ("executed_nodes", EXECUTED_NODES), + ("node_sensors", NODE_SENSORS), + ("warehouse_freshness", WAREHOUSE_FRESHNESS), + ("warehouse_fingerprints", WAREHOUSE_FINGERPRINTS), + ("development_locations", DEVELOPMENT_LOCATIONS), + ("node_transitions", NODE_TRANSITIONS), + ("password_credentials", PASSWORD_CREDENTIALS), + ("password_sessions", PASSWORD_SESSIONS), + ("authentication_revision", AUTHENTICATION_REVISION), + ("oidc_login_flows", OIDC_LOGIN_FLOWS), + ("administrator_initialization", ADMINISTRATOR_INITIALIZATION), + ("external_sessions", EXTERNAL_SESSIONS), + ("warehouse_reconciliation", WAREHOUSE_RECONCILIATION), + ("audit_storage_order", AUDIT_STORAGE_ORDER), + ("warehouse_audit_results", WAREHOUSE_AUDIT_RESULTS), + ("warehouse_janitor_events", WAREHOUSE_JANITOR_EVENTS), + ("warehouse_migrations", WAREHOUSE_MIGRATIONS), + ("warehouse_microbatches", WAREHOUSE_MICROBATCHES), + ("source_sensors", SOURCE_SENSORS), + ("source_staleness", SOURCE_STALENESS), + ("security_audit_sink", SECURITY_AUDIT_SINK), + ("warehouse_catalog", WAREHOUSE_CATALOG), + ("fingerprint_lookup", FINGERPRINT_LOOKUP), + ("warehouse_statistics", WAREHOUSE_STATISTICS), + ("check_observations", CHECK_OBSERVATIONS), + ("trigger_progress", TRIGGER_PROGRESS), +]; + +pub(crate) fn migration_digests(store: &ControlStore) -> Vec { + let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_schema(store).join(";\n")).as_bytes()); + let mut digests = vec![digest]; + digests.extend(ADDITIONS.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); + digests +} pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { let current = Version::parse(env!("CARGO_PKG_VERSION")) diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 12150ca8ed..aa8bc13b98 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -1,17 +1,9 @@ //! Forward-only migration under the same lock used by all other control-plane writes. -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::_helpers::migration::{ - ADMINISTRATOR_INITIALIZATION, AUDIT_STORAGE_ORDER, AUTHENTICATION_REVISION, CHECK_OBSERVATIONS, - DEVELOPMENT_LOCATIONS, EXECUTED_NODES, EXTERNAL_SESSIONS, FINGERPRINT_LOOKUP, - LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, NODE_TRANSITIONS, OIDC_LOGIN_FLOWS, - PASSWORD_CREDENTIALS, PASSWORD_SESSIONS, PLATFORM_SINK, RUN_SENSORS, SCHEMA, - SECURITY_AUDIT_SINK, SENSORS, SOURCE_SENSORS, SOURCE_STALENESS, WAREHOUSE_AUDIT_RESULTS, - WAREHOUSE_CATALOG, WAREHOUSE_FINGERPRINTS, WAREHOUSE_FRESHNESS, WAREHOUSE_JANITOR_EVENTS, - WAREHOUSE_MICROBATCHES, WAREHOUSE_MIGRATIONS, WAREHOUSE_RECONCILIATION, WAREHOUSE_STATISTICS, - render, verify_migrations, + ADDITIONS, SCHEMA, SOURCE_SENSORS, migration_digests, render, verify_migrations, }; use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; @@ -34,41 +26,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ) .await?; let recall_statements = recall_schema(store); - let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); - let additions = [ - ("location_instances", LOCATION_INSTANCES), - ("location_work", LOCATION_WORK), - ("platform_sink", PLATFORM_SINK), - ("sensors", SENSORS), - ("run_sensors", RUN_SENSORS), - ("executed_nodes", EXECUTED_NODES), - ("node_sensors", NODE_SENSORS), - ("warehouse_freshness", WAREHOUSE_FRESHNESS), - ("warehouse_fingerprints", WAREHOUSE_FINGERPRINTS), - ("development_locations", DEVELOPMENT_LOCATIONS), - ("node_transitions", NODE_TRANSITIONS), - ("password_credentials", PASSWORD_CREDENTIALS), - ("password_sessions", PASSWORD_SESSIONS), - ("authentication_revision", AUTHENTICATION_REVISION), - ("oidc_login_flows", OIDC_LOGIN_FLOWS), - ("administrator_initialization", ADMINISTRATOR_INITIALIZATION), - ("external_sessions", EXTERNAL_SESSIONS), - ("warehouse_reconciliation", WAREHOUSE_RECONCILIATION), - ("audit_storage_order", AUDIT_STORAGE_ORDER), - ("warehouse_audit_results", WAREHOUSE_AUDIT_RESULTS), - ("warehouse_janitor_events", WAREHOUSE_JANITOR_EVENTS), - ("warehouse_migrations", WAREHOUSE_MIGRATIONS), - ("warehouse_microbatches", WAREHOUSE_MICROBATCHES), - ("source_sensors", SOURCE_SENSORS), - ("source_staleness", SOURCE_STALENESS), - ("security_audit_sink", SECURITY_AUDIT_SINK), - ("warehouse_catalog", WAREHOUSE_CATALOG), - ("fingerprint_lookup", FINGERPRINT_LOOKUP), - ("warehouse_statistics", WAREHOUSE_STATISTICS), - ("check_observations", CHECK_OBSERVATIONS), - ]; - let mut digests = vec![digest.clone()]; - digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); + let digests = migration_digests(store); let current = verify_migrations(&rows, &digests)?; if rows.is_empty() { for statement in SCHEMA @@ -88,9 +46,9 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { (transaction, _) = query(transaction, "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ VALUES (1, 'control_store', $1, $2, $3)", - vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + vec![Parameter::Text(digests[0].clone()), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; } - for (index, (name, sql)) in additions.into_iter().enumerate() { + for (index, &(name, sql)) in ADDITIONS.iter().enumerate() { if index + 1 < rows.len() { continue; } diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index 51a5d55c56..8605413424 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -15,3 +15,4 @@ pub mod read_recall_view; pub(crate) mod refresh_clock; pub(crate) mod store_source_text; pub(crate) mod subscribe_changes; +pub(crate) mod verify_store; diff --git a/crates/sqlbuild-orchestration/src/store/main/verify_store.rs b/crates/sqlbuild-orchestration/src/store/main/verify_store.rs new file mode 100644 index 0000000000..eecc069bc3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/verify_store.rs @@ -0,0 +1,26 @@ +//! Validate the complete initialized schema without migration, seeding or other writes. + +use crate::store::_helpers::migration::{migration_digests, verify_migrations}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, ControlTransaction}; + +pub(crate) async fn verify_store( + store: &ControlStore, + transaction: ControlTransaction, +) -> Result { + let digests = migration_digests(store); + let (transaction, rows) = query( + transaction, + "SELECT version, checksum, min_server_version FROM sqb_schema_migrations ORDER BY version", + vec![], + ) + .await?; + verify_migrations(&rows, &digests)?; + if rows.len() != digests.len() { + return Err(ControlError::Schema( + "projection repair requires an initialized, fully upgraded control store".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/trigger_progress.sql b/crates/sqlbuild-orchestration/src/store/trigger_progress.sql new file mode 100644 index 0000000000..5672ce6c0c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/trigger_progress.sql @@ -0,0 +1,8 @@ +CREATE TABLE sqb_trigger_progress ( + storage_id BIGINT PRIMARY KEY, + trigger_id {uuid} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + schema_version BIGINT NOT NULL, + projection_json {json} NOT NULL +); +CREATE INDEX sqb_trigger_progress_latest ON sqb_trigger_progress(trigger_id, storage_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs index c95b8ab3a9..7af5cb9637 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs @@ -7,15 +7,20 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; -use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{ - Change, ControlEvent, ControlStore, ControlTransaction, Parameter, TransactionMode, +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; +#[cfg(test)] +use crate::store::{ + main::{begin::begin, commit::commit}, + models::{ControlStore, TransactionMode}, }; use crate::warehouse_metadata::_helpers::ingestion::validate_fields; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +#[cfg(test)] +use crate::warehouse_metadata::main::rebuild_catalog::replay_catalog_projection; use crate::warehouse_metadata::models::{ - CatalogBatch, CatalogObservation, CatalogPage, CatalogQuery, CatalogReplayRow, - WarehouseCatalogRecord, WarehouseReconciliationCursor, + CatalogBatch, CatalogObservation, CatalogPage, CatalogQuery, WarehouseCatalogRecord, + WarehouseReconciliationCursor, }; pub(crate) fn validate_catalog(batch: &CatalogBatch) -> Result<(), ControlError> { @@ -165,54 +170,8 @@ pub(crate) async fn read_catalog_page( } /// Internal replay owner; scan receipts, reader sessions and authoritative facts are preserved. -#[cfg_attr( - not(test), - expect(dead_code, reason = "awaiting the complete rebuild command") -)] +#[cfg(test)] pub(crate) async fn rebuild_catalog_projection(store: &ControlStore) -> Result<(), ControlError> { let transaction = begin(store, TransactionMode::Write).await?; - let (mut transaction, _) = - query(transaction, "DELETE FROM sqb_warehouse_catalog", vec![]).await?; - let mut after = 0; - loop { - let rows; - (transaction, rows) = query(transaction, - "SELECT storage_id,event_schema_version AS schema_version,occurred_at,payload FROM recall_events \ - WHERE producer='warehouse' AND event_type='warehouse_catalog_changed' AND storage_id>$1 \ - ORDER BY storage_id LIMIT 100", vec![Parameter::Integer(after)]).await?; - if rows.is_empty() { - break; - } - for row in rows { - let row: CatalogReplayRow = decode_row(row)?; - if row.schema_version != 1 { - return Err(ControlError::Invalid( - "cannot replay an unsupported catalog fact schema".into(), - )); - } - after = row.storage_id; - transaction.now = row.occurred_at; - let fact = row.payload; - if fact.present { - transaction = project_catalog_record( - transaction, - (fact.location_id, &fact.target), - fact.relation_id, - &fact.record, - ) - .await?; - } else { - (transaction, _) = query( - transaction, - "DELETE FROM sqb_warehouse_catalog WHERE event_id=$1", - vec![Parameter::Id(fact.relation_id)], - ) - .await?; - } - transaction - .changes - .insert(Change::NodeStatus(fact.location_id, fact.target)); - } - } - commit(transaction).await + commit(replay_catalog_projection(transaction).await?).await } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index c20457c46b..8fac735025 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -14,4 +14,5 @@ pub mod read_person_node_status; pub mod read_person_statistics; pub mod read_statistics_due; pub mod read_watermark; +pub(crate) mod rebuild_catalog; pub mod run_statistics_retention; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/rebuild_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/rebuild_catalog.rs new file mode 100644 index 0000000000..991b2bf3ac --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/rebuild_catalog.rs @@ -0,0 +1,56 @@ +//! Restore structural catalog views without changing scan receipts or source facts. + +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::catalog::project_catalog_record; +use crate::warehouse_metadata::models::CatalogReplayRow; + +pub(crate) async fn replay_catalog_projection( + transaction: ControlTransaction, +) -> Result { + let (mut transaction, _) = + query(transaction, "DELETE FROM sqb_warehouse_catalog", vec![]).await?; + let mut after = 0; + loop { + let rows; + (transaction, rows) = query(transaction, + "SELECT storage_id,event_schema_version AS schema_version,occurred_at,payload FROM recall_events \ + WHERE producer='warehouse' AND event_type='warehouse_catalog_changed' AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", vec![Parameter::Integer(after)]).await?; + if rows.is_empty() { + break; + } + for row in rows { + let row: CatalogReplayRow = decode_row(row)?; + if row.schema_version != 1 { + return Err(ControlError::Invalid( + "cannot replay an unsupported catalog fact schema".into(), + )); + } + after = row.storage_id; + transaction.now = row.occurred_at; + let fact = row.payload; + if fact.present { + transaction = project_catalog_record( + transaction, + (fact.location_id, &fact.target), + fact.relation_id, + &fact.record, + ) + .await?; + } else { + (transaction, _) = query( + transaction, + "DELETE FROM sqb_warehouse_catalog WHERE event_id=$1", + vec![Parameter::Id(fact.relation_id)], + ) + .await?; + } + transaction + .changes + .insert(Change::NodeStatus(fact.location_id, fact.target)); + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs index b1cb6bd08a..81aa3e85fd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/replay/helpers.rs @@ -1,12 +1,18 @@ //! Full-state acceptance keeps protected live ownership and source facts non-vacuous. -use crate::runs::_helpers::lifecycle::rebuild::rebuild_run_projections; +use crate::runs::main::replay_projections::replay_run_projections; +use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::logical_state::logical_state; +pub(crate) async fn rebuild_run_projections(store: &ControlStore) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + commit(replay_run_projections(transaction).await?).await +} + pub(crate) async fn run_replay_contract(store: &ControlStore) { let expected = logical_state(store).await; let mut transaction = begin(store, TransactionMode::Write) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_replay.rs b/crates/sqlbuild-orchestration/tests/runs/test_replay.rs index a652652ae9..ddce3f206e 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_replay.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_replay.rs @@ -3,12 +3,11 @@ use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::runs::_helpers::lifecycle::change_run::change_run_in; -use crate::runs::_helpers::lifecycle::rebuild::rebuild_run_projections; use crate::runs::_helpers::requests::publication::publish_request; use crate::runs::main::list_runs::list_runs; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunStatus}; use crate::runs::tests::helpers::{queue, request, snapshot}; -use crate::runs::tests::replay_helpers::run_replay_contract; +use crate::runs::tests::replay_helpers::{rebuild_run_projections, run_replay_contract}; use crate::runs::tests::test_types::RunReplayTestCase; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::{ diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs new file mode 100644 index 0000000000..522d574335 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs @@ -0,0 +1,246 @@ +//! Reset/re-evaluation records the distinguishing fact even when every tick already exists. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::platform::main::rebuild_projections::rebuild_projections; +use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::scheduler::tests::contracts::ticks::test_types::TriggerControlsTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_reset_existing_ticks_when_progress_is_replayed_then_cursor_and_due_time_are_exact() { + let test_cases = [ + TriggerControlsTestCase { + description: "SQLite exact semantic progress and atomic repair", + backend: Backend::Sqlite, + expected_events: 4, + }, + TriggerControlsTestCase { + description: "PostgreSQL exact semantic progress and atomic repair", + backend: Backend::Postgres, + expected_events: 4, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publication"); + let id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + evaluate_trigger(store, &lease, id) + .await + .expect("first drain"); + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + change_trigger( + store, + &audit, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id: id, + actor: audit.actor.clone(), + reason: "Replay retained calendar".into(), + change: TriggerChange::ResetCursor, + }, + ) + .await + .expect("reset"); + let reset = list_triggers(store, location, "development") + .await + .expect("reset state") + .remove(0); + assert!(reset.last_tick_for.is_none()); + assert_eq!( + reset.next_due_at.expect("first due").to_rfc3339(), + "2020-01-01T00:00:00+00:00" + ); + let fault = TransactionFault { + description: "progress publication failure", + table: "sqb_trigger_progress", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'synthetic progress failure')", + postgres_body: "RAISE EXCEPTION 'synthetic progress failure'", + }; + install_fault(store, &fault).await; + let before = logical_state(store).await; + let error = evaluate_trigger(store, &lease, id) + .await + .expect_err("fact failure rejects cursor update"); + assert!(error.to_string().contains("synthetic progress failure")); + assert_eq!( + logical_state(store).await, + before, + "entire transaction rolled back" + ); + remove_fault(store, &fault).await; + evaluate_trigger(store, &lease, id) + .await + .expect("re-evaluate retained ticks"); + let expected = list_triggers(store, location, "development") + .await + .expect("advanced state"); + assert_eq!( + expected[0].last_tick_for.expect("last due").to_rfc3339(), + "2020-01-03T00:00:00+00:00" + ); + assert_eq!( + expected[0].next_due_at.expect("future due").to_rfc3339(), + "2030-01-01T00:00:00+00:00" + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("source cut"); + let (transaction, facts) = query( + transaction, + "SELECT projection_json FROM sqb_trigger_progress ORDER BY storage_id", + vec![], + ) + .await + .expect("progress"); + assert_eq!( + facts.len(), + test_case.expected_events, + "{}: publish, drain, reset, drain despite duplicate ticks", + test_case.description + ); + assert_eq!(facts[2]["projection_json"]["last_tick_for"], json!(null)); + assert_eq!( + facts[3]["projection_json"]["last_tick_for"], + json!("2020-01-03T00:00:00Z") + ); + commit(transaction).await.expect("source cut complete"); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("backward clock transition"); + transaction.now = "1999-01-01T00:00:00Z" + .parse() + .expect("synthetic database clock"); + let pause = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id: id, + actor: audit.actor.clone(), + reason: "Pause after clock step".into(), + change: TriggerChange::Pause, + }; + let (transaction, applied) = change_trigger_in(transaction, &audit, &pause) + .await + .expect("causally later pause"); + assert!(applied); + commit(transaction) + .await + .expect("clock-step fact and state"); + let expected = list_triggers(store, location, "development") + .await + .expect("causally latest paused trigger"); + assert_eq!( + expected[0].status, + crate::declarations::models::TriggerStatus::Paused + ); + let original = logical_state(store).await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("damage"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/projection_replay.json"); + let damage: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("damage fixture")) + .expect("damage cases"); + for sql in damage["trigger_damage"].as_array().expect("trigger damage") { + (transaction, _) = query(transaction, sql.as_str().expect("damage SQL"), vec![]) + .await + .expect("corrupt projections"); + } + commit(transaction).await.expect("commit damage"); + let damaged = logical_state(store).await; + let late_fault = TransactionFault { + description: "late run replay failure after trigger repair", + table: "sqb_run_nodes", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'synthetic late replay failure')", + postgres_body: "RAISE EXCEPTION 'synthetic late replay failure'", + }; + install_fault(store, &late_fault).await; + assert!( + rebuild_projections(store) + .await + .expect_err("late failure") + .to_string() + .contains("synthetic late replay failure") + ); + assert_eq!( + logical_state(store).await, + damaged, + "no earlier phase commits" + ); + remove_fault(store, &late_fault).await; + rebuild_projections(store).await.expect("atomic replay"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("restored"), + expected + ); + assert_eq!( + logical_state(store).await, + original, + "all protected rows and original timestamps retained" + ); + rebuild_projections(store).await.expect("idempotent replay"); + assert_eq!(logical_state(store).await, original); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("legacy source cut"); + let (transaction, _) = query(transaction, "DELETE FROM sqb_trigger_progress", vec![]) + .await + .expect("legacy history contains no progress facts"); + let (transaction, _) = query(transaction, "DELETE FROM sqb_current_snapshots", vec![]) + .await + .expect("earlier repair needed"); + commit(transaction).await.expect("legacy fixture"); + let legacy = logical_state(store).await; + let error = rebuild_projections(store) + .await + .expect_err("ambiguous reset/re-evaluation cannot be guessed"); + assert!(error.to_string().contains("lacks semantic progress")); + assert_eq!( + logical_state(store).await, + legacy, + "legacy refusal rolls back current pointers too" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index 1b9c0ad341..2e582f75d0 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -4,6 +4,8 @@ pub(crate) mod helpers; mod test_contract; #[path = "test_loop.rs"] mod test_loop; +#[path = "test_progress.rs"] +mod test_progress; #[path = "test_triggers.rs"] mod test_triggers; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs index 426fc861a4..ad34ab65c2 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{TriggerChange, TriggerStatus}; +use crate::platform::main::rebuild_projections::rebuild_projections; use crate::runs::main::list_runs::list_runs; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; @@ -13,8 +14,12 @@ use crate::sensors::tests::helpers::{append_events, change, evaluations, snapsho use crate::sensors::tests::test_types::SensorTestCase; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; -use crate::store::main::migrate_store::migrate_store; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::logical_state::logical_state; use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -199,6 +204,20 @@ fn given_a_crashed_sensor_when_leadership_or_snapshot_changes_then_stale_results .expect("interval measured from start, not completion") .expect("already due"); assert_ne!(immediate.evaluation_id, delayed.evaluation_id); + let expected = logical_state(store).await; + let transaction = begin(store, TransactionMode::Write).await.expect("damage"); + let (transaction, _) = query(transaction, "DELETE FROM sqb_trigger_state", vec![]) + .await + .expect("missing sensor projection"); + commit(transaction).await.expect("damage committed"); + rebuild_projections(store) + .await + .expect("repair with pending evaluation and live cursor"); + assert_eq!( + logical_state(store).await, + expected, + "replay preserves attempts, cursor commits, deduplication and operational ownership" + ); complete_builtin_sensor(store, &successor, &immediate) .await .expect("complete next attempt"); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index f700fcde8d..f90e6991bf 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -4,11 +4,12 @@ use std::collections::BTreeMap; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::rebuild_projections::replay_snapshot_projections; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; -use crate::store::main::migrate_store::migrate_store; use crate::store::main::open_store::open_store; -use crate::store::models::ControlStore; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::{ControlStore, TransactionMode}; use serde_json::json; use uuid::Uuid; @@ -20,6 +21,11 @@ use crate::store::tests::store_helpers::{ }; use crate::store::tests::store_test_types::Backend; +pub(crate) async fn rebuild_snapshot_projections(store: &ControlStore) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + commit(replay_snapshot_projections(transaction).await?).await +} + pub(super) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { location_id, diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs index d45abb7b61..6ad5ff633b 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs @@ -5,10 +5,9 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; -use crate::snapshots::_helpers::rebuild::rebuild_snapshot_projections; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; -use crate::snapshots::tests::helpers::{damage, snapshot}; +use crate::snapshots::tests::helpers::{damage, rebuild_snapshot_projections, snapshot}; use crate::snapshots::tests::test_types::SnapshotTestCase; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs index 08fa4ca5be..a6e1ca4aed 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_replay_writers.rs @@ -1,7 +1,7 @@ //! Replay and normal publication share the writer fence while readers keep a consistent cut. use crate::credentials::tests::location_helpers::create_location; -use crate::snapshots::_helpers::rebuild::rebuild_snapshot_projections; +use crate::platform::main::rebuild_projections::rebuild_projections; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::tests::helpers::snapshot; @@ -62,7 +62,7 @@ fn given_concurrent_publication_when_replaying_then_readers_and_writers_keep_one let repair_store = replica.clone(); let mut repair = tokio::spawn(async move { ready.send(()).expect("repair ready"); - rebuild_snapshot_projections(&repair_store).await + rebuild_projections(&repair_store).await }); started.await.expect("repair started"); assert!( @@ -118,9 +118,7 @@ fn given_concurrent_publication_when_replaying_then_readers_and_writers_keep_one "Current" ); let final_state = logical_state(&replica).await; - rebuild_snapshot_projections(store) - .await - .expect("replay newest cut"); + rebuild_projections(store).await.expect("replay newest cut"); assert_eq!(logical_state(&replica).await, final_state); close_store(replica).await; close_test_store(fixture).await; diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 63af06e2d7..55d56b7fd8 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -92,6 +92,8 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DROP TABLE sqb_trigger_progress", + "DELETE FROM sqb_schema_migrations WHERE name='trigger_progress'", "DROP INDEX sqb_observed_node_cursor", "DROP INDEX sqb_observed_node_identity", "DROP INDEX sqb_observed_node_decisive", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 35373fb57b..131fd6acea 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -114,6 +114,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb init](cli/init.md) (`cli/init`) - Scaffold a new SQLBuild project. - [sqb playground](cli/playground.md) (`cli/playground`) - Create a self-contained SQLBuild project to explore locally. - [sqb skills](cli/skills.md) (`cli/skills`) - Install SQLBuild skill files for AI coding agents. +- [sqb server](cli/server.md) (`cli/server`) - Run the control service or repair its derived views. - [sqb compile](cli/compile.md) (`cli/compile`) - Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. - [sqb contract](cli/contract.md) (`cli/contract`) - Compare or generate repository contracts from physical warehouse schemas. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md new file mode 100644 index 0000000000..c3f165ff54 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -0,0 +1,61 @@ + + +# sqb server + +> Run the control service or repair its derived views. + +Online: https://sqlbuild.com/docs/cli/server/ + +Run the control API and scheduler without loading project Python: + +```bash +sqb server +``` + +Use `sqb server --help` for service, authentication and launcher options. + +## Repair control views + +Repair snapshot selection, jobs, triggers, runs, node status and the structural catalog from +retained control facts: + +```bash +sqb server rebuild-projections +``` + +This operation requires direct ownership of the initialized control database. A browser role +or location credential does not grant it. Back up the database before repair. This command +does not initialize or upgrade a database; start the matching server version first if a schema +upgrade is needed. + +| Database | Selection | +| --- | --- | +| SQLite | `.sqlbuild/control.sqlite3` in the project directory when the database variable is unset | +| PostgreSQL | URL in `SQLBUILD_CONTROL_DATABASE_URL` | + +Use `--database-env VARIABLE` to read the database URL from a different environment variable. +Use `--project-dir PATH` to select a different SQLite project directory. + +Repair waits for the current writer and uses one consistent source cut. It commits all repaired +views together or leaves every view unchanged. Repeating a successful repair has the same result. +It preserves credentials, security audits, retained source facts, worker claims, leases and other +operational ownership. It does not rerun jobs, release workers or modify the warehouse. + +## Handle a refused repair + +Repair fails if a retained source is unsupported, invalid or insufficient to reconstruct the +views exactly. A trigger without a recorded semantic-progress image cannot be repaired: tick +history alone cannot distinguish a cursor reset from a subsequent reevaluation of existing ticks. +The command never guesses the cursor, substitutes the current time or reports partial success. + +If repair reports missing semantic progress, preserve the database and use a known-good backup. +Do not delete source history to force a repair. Recording future transitions does not recover +missing historical facts. + +| Result | Exit status | stdout | +| --- | --- | --- | +| All views repaired | `0` | `{"rebuilt":true}` | +| Repair refused or rolled back | Nonzero | No success result | + +Progress and completion messages go to stderr. If stdout delivery fails after commit, the +command reports that changes may already be committed; rerunning the repair is safe. diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 0baeaa122a..f26ad2ab8f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -262,3 +262,16 @@ class StatisticsRecoveryTestCase: class DenialMetadataTestCase: description: str expected_denial_count: int + + +class ProjectionStore(NamedTuple): + environment: dict[str, str] + connection: Any + table_query: str + + +@dataclass(frozen=True) +class ProjectionReplayTestCase: + description: str + open_store: Callable[[tuple[Path, Path]], AbstractContextManager[ProjectionStore]] + expected_progress_facts: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/replay/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/replay/helpers.py new file mode 100644 index 0000000000..bdd1c81576 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/replay/helpers.py @@ -0,0 +1,63 @@ +"""Real initialized stores and installed repair processes for projection acceptance.""" + +import sqlite3 +import subprocess +from collections.abc import Iterator +from contextlib import closing, contextmanager +from pathlib import Path + +import psycopg +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ProjectionStore, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + environment, + serve, +) + + +@contextmanager +def sqlite_store(installation: tuple[Path, Path]) -> Iterator[ProjectionStore]: + """Initialize through the real server before opening a diagnostic connection.""" + with serve(installation): + pass + with closing(sqlite3.connect(installation[1] / ".sqlbuild/control.sqlite3")) as connection: + yield ProjectionStore( + {}, + connection, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'", + ) + + +@contextmanager +def postgres_store(installation: tuple[Path, Path]) -> Iterator[ProjectionStore]: + """Own the disposable database independently of every installed server/repair process.""" + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + extra: dict[str, str] = {"SQLBUILD_CONTROL_DATABASE_URL": postgres.get_connection_url()} + with serve(installation, extra_environment=extra): + pass + with psycopg.connect(extra["SQLBUILD_CONTROL_DATABASE_URL"]) as connection: + yield ProjectionStore( + extra, + connection, + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema=current_schema() AND table_type='BASE TABLE'", + ) + + +def repair( + installation: tuple[Path, Path], extra: dict[str, str] +) -> subprocess.CompletedProcess[str]: + """Run the installed maintenance command with isolated database selection.""" + return subprocess.run( + [str(installation[0]), "server", "rebuild-projections"], + cwd=installation[1], + env=environment(extra), + text=True, + capture_output=True, + check=False, + timeout=30, + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_projection_rebuild.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_projection_rebuild.py new file mode 100644 index 0000000000..685388c276 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_projection_rebuild.py @@ -0,0 +1,140 @@ +"""Installed repair commits every projection or none on both supported stores.""" + +import json +import shutil +import subprocess +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ProjectionReplayTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.replay.helpers import ( + postgres_store, + repair, + sqlite_store, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + serve, + successful, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers import ( + logical_table_state, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectionReplayTestCase("SQLite installed atomic repair", sqlite_store, 4), + ProjectionReplayTestCase("PostgreSQL installed atomic repair", postgres_store, 4), + ], + ids=lambda case: case.description, +) +def test_given_damaged_projections_when_the_installed_command_repairs_then_sources_and_failure_atomicity_hold( + test_case: ProjectionReplayTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + write_files(project_dir=project_dir, files=files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + with test_case.open_store(installation) as store: + location: str = successful( + administer( + installation, ["create-location", "orders"], extra_environment=store.environment + ) + )["location_id"] + token: dict[str, Any] = successful( + administer( + installation, + [ + "issue-location-token", + location, + "--expires-at", + (datetime.now(UTC) + timedelta(hours=1)).isoformat(), + ], + extra_environment=store.environment, + ) + ) + with serve(installation, extra_environment=store.environment) as url: + snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, + json.dumps(METADATA | {"location_id": location, "target": "development"}), + project_dir, + ) + _ = snapshot.publish(url, token["bearer"]) + connection: Any = store.connection + tables: list[str] = [row[0] for row in connection.execute(store.table_query).fetchall()] + progress_count: Any = connection.execute( + "SELECT COUNT(*) FROM sqb_trigger_progress" + ).fetchone() + assert progress_count[0] == test_case.expected_progress_facts + original: dict[str, str] = logical_table_state(connection, tables) + damage: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/projection_replay.json").read_text() + ) + for statement in damage["snapshot_damage"] + damage["trigger_damage"]: + _ = connection.execute( + statement.replace("job_key='orders'", "job_key='edge_snapshot_orders'") + ) + connection.commit() + for _ in range(2): + output: subprocess.CompletedProcess[str] = repair(installation, store.environment) + assert successful(output) == {"rebuilt": True} + assert "Rebuilding control projections" in output.stderr + assert "rebuilt successfully" in output.stderr + assert logical_table_state(connection, tables) == original + _ = connection.execute( + "UPDATE sqb_schema_migrations SET min_server_version='9999.0.0' WHERE name='trigger_progress'" + ) + connection.commit() + incompatible: dict[str, str] = logical_table_state(connection, tables) + output = repair(installation, store.environment) + assert output.returncode != 0 and not output.stdout + assert "minimum server version not met" in output.stderr + assert logical_table_state(connection, tables) == incompatible + _ = connection.execute( + "UPDATE sqb_schema_migrations SET min_server_version='0.0.0' WHERE name='trigger_progress'" + ) + _ = connection.execute("DELETE FROM sqb_trigger_progress") + _ = connection.execute("DELETE FROM sqb_current_snapshots") + connection.commit() + legacy: dict[str, str] = logical_table_state(connection, tables) + output = repair(installation, store.environment) + assert output.returncode != 0 and not output.stdout + assert "lacks semantic progress" in output.stderr + assert "failed" in output.stderr + assert logical_table_state(connection, tables) == legacy diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py index e94c59a59b..dc12c5b047 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py @@ -21,6 +21,19 @@ ) +def logical_table_state(connection: Any, tables: list[str]) -> dict[str, str]: + """Hash complete logical rows and multiplicity without exposing stored credentials.""" + state: dict[str, str] = {} + for table in tables: + rows: list[str] = [ + json.dumps(row, sort_keys=True, default=str) + for row in connection.execute(f'SELECT * FROM "{table}"').fetchall() + ] + state[table] = hashlib.sha256(json.dumps(sorted(rows)).encode()).hexdigest() + connection.commit() + return state + + def postgres_recovery_state(connection: Any) -> dict[str, str]: """Compare every retained security row without logging credential hashes.""" tables: tuple[str, ...] = ( diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx new file mode 100644 index 0000000000..917fa72103 --- /dev/null +++ b/website/src/content/docs/docs/cli/server.mdx @@ -0,0 +1,60 @@ +--- +title: "sqb server" +description: "Run the control service or repair its derived views." +sidebar: + label: "server" +--- + +Run the control API and scheduler without loading project Python: + +```bash +sqb server +``` + +Use `sqb server --help` for service, authentication and launcher options. + +## Repair control views + +Repair snapshot selection, jobs, triggers, runs, node status and the structural catalog from +retained control facts: + +```bash +sqb server rebuild-projections +``` + +This operation requires direct ownership of the initialized control database. A browser role +or location credential does not grant it. Back up the database before repair. This command +does not initialize or upgrade a database; start the matching server version first if a schema +upgrade is needed. + +| Database | Selection | +| --- | --- | +| SQLite | `.sqlbuild/control.sqlite3` in the project directory when the database variable is unset | +| PostgreSQL | URL in `SQLBUILD_CONTROL_DATABASE_URL` | + +Use `--database-env VARIABLE` to read the database URL from a different environment variable. +Use `--project-dir PATH` to select a different SQLite project directory. + +Repair waits for the current writer and uses one consistent source cut. It commits all repaired +views together or leaves every view unchanged. Repeating a successful repair has the same result. +It preserves credentials, security audits, retained source facts, worker claims, leases and other +operational ownership. It does not rerun jobs, release workers or modify the warehouse. + +## Handle a refused repair + +Repair fails if a retained source is unsupported, invalid or insufficient to reconstruct the +views exactly. A trigger without a recorded semantic-progress image cannot be repaired: tick +history alone cannot distinguish a cursor reset from a subsequent reevaluation of existing ticks. +The command never guesses the cursor, substitutes the current time or reports partial success. + +If repair reports missing semantic progress, preserve the database and use a known-good backup. +Do not delete source history to force a repair. Recording future transitions does not recover +missing historical facts. + +| Result | Exit status | stdout | +| --- | --- | --- | +| All views repaired | `0` | `{"rebuilt":true}` | +| Repair refused or rolled back | Nonzero | No success result | + +Progress and completion messages go to stderr. If stdout delivery fails after commit, the +command reports that changes may already be committed; rerunning the repair is safe. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index c8f1f25bdf..2be1e116b7 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -200,6 +200,7 @@ "docs/cli/init", "docs/cli/playground", "docs/cli/skills", + "docs/cli/server", "docs/cli/compile", "docs/cli/format", "docs/cli/contract", From 9bfefdf6b2ce96da636cf5e66fb4bf4bf925c229 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:45:02 +0000 Subject: [PATCH 118/138] feat: add browser-authorized CLI person sessions --- Cargo.lock | 775 +++++++++++++++++- Cargo.toml | 1 + .../edge_cases/orchestration/cli_login.json | 6 + crates/sqlbuild-cli/Cargo.toml | 2 + .../sqlbuild-cli/src/argv/command_table.json | 30 + .../src/launcher/_helpers/dispatch.rs | 6 + crates/sqlbuild-cli/src/lib.rs | 1 + .../src/native_commands/main/_run_admin.rs | 12 +- .../src/native_commands/main/_run_dev.rs | 2 +- .../src/native_commands/main/mod.rs | 1 + .../main/run_login_invocation.rs | 48 ++ .../src/person_login/_helpers/credentials.rs | 200 +++++ .../src/person_login/_helpers/flow.rs | 117 +++ .../src/person_login/_helpers/mod.rs | 3 + .../src/person_login/_helpers/transport.rs | 101 +++ .../src/person_login/constants.rs | 7 + .../sqlbuild-cli/src/person_login/main/mod.rs | 1 + .../src/person_login/main/run_login.rs | 70 ++ crates/sqlbuild-cli/src/person_login/mod.rs | 6 + .../sqlbuild-cli/src/person_login/models.rs | 58 ++ .../src/workers/_helpers/supervisor.rs | 2 +- .../tests/native_commands/worker/helpers.rs | 6 +- .../src/api/_helpers/authentication/bearer.rs | 2 +- .../_helpers/authentication/cli_login.html | 48 ++ .../api/_helpers/authentication/cli_login.js | 78 ++ .../api/_helpers/authentication/cli_login.rs | 122 +++ .../src/api/_helpers/authentication/mod.rs | 1 + .../src/api/_helpers/authentication/oidc.rs | 29 +- .../_helpers/authentication/oidc_provider.rs | 4 +- .../authentication/password_sessions.rs | 21 +- .../authentication/password_transport.rs | 52 +- .../_helpers/authentication/trusted_proxy.rs | 13 +- .../api/_helpers/resources/administrators.rs | 4 +- .../src/api/_helpers/resources/audit.rs | 2 +- .../src/api/_helpers/resources/changes.rs | 2 +- .../resources/person_location_work.rs | 4 +- .../_helpers/resources/person_operations.rs | 20 +- .../src/api/main/mod.rs | 1 + .../src/api/main/openapi_document.rs | 10 + .../src/api/main/parse_origin.rs | 28 + .../sqlbuild-orchestration/src/api/models.rs | 66 +- .../_helpers/administrator_changes.rs | 2 +- .../_helpers/administrator_roles.rs | 2 +- .../authorization/_helpers/administrators.rs | 2 +- .../authorization/main/authorize_session.rs | 2 +- .../src/authorization/main/bootstrap_user.rs | 4 +- .../authorization/main/deny_authorization.rs | 2 +- .../main/recover_administrator.rs | 4 +- .../credentials/_helpers/active_session.rs | 29 + .../credentials/_helpers/browser_sessions.rs | 2 +- .../src/credentials/_helpers/locations.rs | 2 +- .../src/credentials/_helpers/mod.rs | 1 + .../src/credentials/_helpers/tokens.rs | 2 +- .../main/{ => audit}/append_audit.rs | 0 .../src/credentials/main/audit/mod.rs | 3 + .../record_authentication_denial.rs | 2 +- .../reserve_authentication_attempt.rs | 2 +- .../main/authenticate_session_in.rs | 40 - .../main/cli_login/approve_login.rs | 71 ++ .../credentials/main/cli_login/claim_login.rs | 69 ++ .../src/credentials/main/cli_login/mod.rs | 3 + .../credentials/main/cli_login/start_login.rs | 57 ++ .../main/{ => identities}/create_principal.rs | 2 +- .../main/{ => identities}/insert_principal.rs | 2 +- .../src/credentials/main/identities/mod.rs | 2 + .../main/{ => locations}/create_location.rs | 0 .../{ => locations}/development_location.rs | 0 .../src/credentials/main/locations/mod.rs | 2 + .../src/credentials/main/mod.rs | 26 +- .../authenticate_bound_person.rs | 2 +- .../{ => sessions}/authenticate_session.rs | 2 +- .../main/sessions/authenticate_session_in.rs | 29 + .../main/{ => sessions}/claim_oidc_flow.rs | 2 +- .../main/{ => sessions}/create_oidc_flow.rs | 2 +- .../main/{ => sessions}/login_password.rs | 6 +- .../src/credentials/main/sessions/mod.rs | 9 + .../main/{ => sessions}/revoke_session.rs | 2 +- .../main/{ => sessions}/set_password.rs | 4 +- .../main/{ => sessions}/verify_password.rs | 0 .../main/{ => tokens}/authenticate_token.rs | 0 .../main/{ => tokens}/issue_token.rs | 0 .../src/credentials/main/tokens/mod.rs | 4 + .../main/{ => tokens}/revoke_token.rs | 0 .../main/{ => tokens}/rotate_token.rs | 2 +- .../src/credentials/models.rs | 29 + .../declarations/_helpers/operator_change.rs | 2 +- .../platform/main/request_location_work.rs | 2 +- .../src/runs/_helpers/lifecycle/change_run.rs | 2 +- .../src/runs/_helpers/requests/manual_job.rs | 2 +- .../runs/_helpers/requests/manual_rerun.rs | 2 +- .../src/store/_helpers/migration.rs | 2 + .../src/store/cli_login_flows.sql | 8 + .../tests/api/test_catalog_reads.rs | 3 +- .../tests/api/test_changes.rs | 2 +- .../tests/api/test_cli_login.rs | 396 +++++++++ .../tests/api/test_contract.rs | 17 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_development.rs | 2 +- .../tests/api/test_external_sessions.rs | 4 +- .../tests/api/test_location_health.rs | 2 +- .../tests/api/test_location_work.rs | 2 +- .../tests/api/test_oidc.rs | 2 +- .../tests/api/test_passwords.rs | 4 +- .../tests/api/test_person_location_work.rs | 3 +- .../tests/api/test_person_sessions.rs | 3 +- .../tests/api/test_proxy.rs | 2 +- .../tests/api/test_reads.rs | 6 +- .../tests/api/test_types.rs | 6 + .../tests/api/test_warehouse_metadata.rs | 2 +- .../tests/api/test_worker.rs | 2 +- .../tests/credentials/locations/helpers.rs | 2 +- .../credentials/passwords/test_contract.rs | 4 +- .../credentials/sessions/test_contract.rs | 9 +- .../tests/credentials/test_administrators.rs | 4 +- .../tests/credentials/test_audit.rs | 4 +- .../tests/credentials/test_contract.rs | 12 +- .../tests/credentials/test_recovery.rs | 7 +- .../tests/runs/test_permissions.rs | 3 +- .../tests/sensors/test_custom.rs | 2 +- .../tests/store/helpers.rs | 2 + fensu.toml | 17 +- .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/login.md | 65 ++ .../publication/_test_types.py | 14 + .../publication/login/helpers.py | 351 ++++++++ .../publication/test_cli_login.py | 297 +++++++ .../publication/wheel/cli/helpers.py | 11 +- website/src/content/docs/docs/cli/login.mdx | 64 ++ website/src/sidebar.json | 1 + 129 files changed, 3511 insertions(+), 229 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/cli_login.json create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs create mode 100644 crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs create mode 100644 crates/sqlbuild-cli/src/person_login/_helpers/flow.rs create mode 100644 crates/sqlbuild-cli/src/person_login/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/person_login/_helpers/transport.rs create mode 100644 crates/sqlbuild-cli/src/person_login/constants.rs create mode 100644 crates/sqlbuild-cli/src/person_login/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/person_login/main/run_login.rs create mode 100644 crates/sqlbuild-cli/src/person_login/mod.rs create mode 100644 crates/sqlbuild-cli/src/person_login/models.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs create mode 100644 crates/sqlbuild-orchestration/src/api/main/parse_origin.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => audit}/append_audit.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => audit}/record_authentication_denial.rs (93%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => audit}/reserve_authentication_attempt.rs (98%) delete mode 100644 crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => identities}/create_principal.rs (89%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => identities}/insert_principal.rs (95%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => locations}/create_location.rs (100%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => locations}/development_location.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/authenticate_bound_person.rs (96%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/authenticate_session.rs (87%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session_in.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/claim_oidc_flow.rs (97%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/create_oidc_flow.rs (96%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/login_password.rs (89%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/revoke_session.rs (96%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/set_password.rs (96%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => sessions}/verify_password.rs (100%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => tokens}/authenticate_token.rs (100%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => tokens}/issue_token.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs rename crates/sqlbuild-orchestration/src/credentials/main/{ => tokens}/revoke_token.rs (100%) rename crates/sqlbuild-orchestration/src/credentials/main/{ => tokens}/rotate_token.rs (96%) create mode 100644 crates/sqlbuild-orchestration/src/store/cli_login_flows.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_cli_login.rs create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py create mode 100644 website/src/content/docs/docs/cli/login.mdx diff --git a/Cargo.lock b/Cargo.lock index 3876490943..462a1de8ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -113,6 +124,123 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", +] + +[[package]] +name = "async-recursion" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "atoi" version = "2.0.0" @@ -341,6 +469,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -350,6 +487,19 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "borsh" version = "1.8.1" @@ -421,6 +571,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.4" @@ -478,6 +637,16 @@ dependencies = [ "phf 0.12.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "cmov" version = "0.5.4" @@ -524,6 +693,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -542,6 +720,26 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -807,6 +1005,36 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "aes", + "block-padding", + "cbc", + "dbus", + "fastrand", + "hkdf", + "num", + "once_cell", + "openssl", + "sha2 0.10.9", + "zeroize", +] + [[package]] name = "defmt" version = "1.2.0" @@ -1051,6 +1279,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1088,6 +1343,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1164,6 +1429,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1226,6 +1506,30 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "futures-sink" version = "0.3.34" @@ -1246,6 +1550,7 @@ checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1417,6 +1722,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -1743,6 +2054,16 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "interpolator" version = "0.5.0" @@ -1866,6 +2187,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "byteorder", + "dbus-secret-service", + "log", + "openssl", + "secret-service", + "security-framework 2.11.1", + "security-framework 3.7.0", + "windows-sys 0.60.2", + "zeroize", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -1881,6 +2219,16 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.16" @@ -2059,6 +2407,19 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "nix" version = "0.31.3" @@ -2081,6 +2442,20 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -2107,6 +2482,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2123,11 +2507,22 @@ dependencies = [ ] [[package]] -name = "num-iter" -version = "0.1.46" +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ + "num-bigint", "num-integer", "num-traits", ] @@ -2233,6 +2628,53 @@ dependencies = [ "url", ] +[[package]] +name = "openssl" +version = "0.10.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de8111e94a56adca7ebfe6c365b6d42f99deb69b74b6becacd4204e2f193aa2" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c413d6ec9ebaf5e1c829d355cb3e9f783778d700f135be0921d52fce3c60ae" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -2242,6 +2684,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + [[package]] name = "ordermap" version = "1.2.2" @@ -2414,6 +2866,17 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -2457,6 +2920,20 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -3295,6 +3772,61 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secret-service" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" +dependencies = [ + "aes", + "cbc", + "futures-util", + "generic-array", + "hkdf", + "num", + "once_cell", + "rand 0.8.8", + "serde", + "sha2 0.10.9", + "zbus", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -3375,6 +3907,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -3439,6 +3982,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -3647,6 +4201,7 @@ dependencies = [ "command-fds", "ctrlc", "getrandom 0.4.3", + "keyring", "reqwest", "rpassword", "rusqlite", @@ -3661,6 +4216,7 @@ dependencies = [ "sqlbuild-cursor", "sqlbuild-model-config", "sqlbuild-orchestration", + "sqlbuild-recall", "sqlbuild-rules", "tempfile", "tokio", @@ -4454,6 +5010,17 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys 0.61.2", +] + [[package]] name = "unarray" version = "0.1.4" @@ -4870,6 +5437,24 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -4903,13 +5488,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4922,6 +5524,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4934,6 +5542,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4946,12 +5560,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4964,6 +5590,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4976,6 +5608,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4988,6 +5626,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -5000,6 +5644,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.15" @@ -5057,6 +5707,16 @@ dependencies = [ "time", ] +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "yasna" version = "0.6.0" @@ -5090,6 +5750,62 @@ dependencies = [ "synstructure 0.14.0", ] +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-process", + "async-recursion", + "async-trait", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "rand 0.8.8", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant", +] + [[package]] name = "zerocopy" version = "0.8.62" @@ -5136,6 +5852,20 @@ name = "zeroize" version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" @@ -5181,3 +5911,40 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "zvariant_derive", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/Cargo.toml b/Cargo.toml index c6500ba2c0..adb985a369 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ getrandom = "0.4" globset = "0.4" # Windows only: tells NTFS junctions, which Python walks as directories, from symbolic links. junction = { version = "2.1.0", default-features = false } +keyring = { version = "3.6.3", default-features = false } # Python loads the extension with dlopen, so mimalloc must not use initial-exec TLS. mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } diff --git a/benchmarks/duckshop/edge_cases/orchestration/cli_login.json b/benchmarks/duckshop/edge_cases/orchestration/cli_login.json new file mode 100644 index 0000000000..76b1e580fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/cli_login.json @@ -0,0 +1,6 @@ +{ + "origin": "https://control.example.com", + "password": "orders-browser-password", + "replacement": "orders-replacement-password", + "client_secret": "synthetic-cli-exchange-capability-not-a-real-credential" +} diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 6e46d7a345..1f24eb892c 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -19,6 +19,7 @@ path = "src/bin/sqlbuild.rs" blake3.workspace = true chrono.workspace = true getrandom.workspace = true +keyring = { workspace = true, features = ["apple-native", "windows-native", "sync-secret-service", "crypto-rust", "vendored"] } reqwest.workspace = true rpassword.workspace = true serde.workspace = true @@ -30,6 +31,7 @@ sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true sqlbuild-model-config.workspace = true sqlbuild-orchestration.workspace = true +sqlbuild-recall.workspace = true sqlbuild-rules.workspace = true tokio.workspace = true uuid.workspace = true diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index dff1f9d3c2..7f8b9949d1 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,34 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "login", + "parser": { + "prog": "sqb login", + "description": "Sign in through your browser, or verify a saved live CLI session. Credentials stay in your user credential store, never in the project or terminal output.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "dest": "login_action", "nargs": "?", "choices": ["status"], "help": "verify the saved credential with the live server", "group": 0}, + {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin", "group": 1}, + {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1}, + {"kind": "store_true", "options": ["--no-browser"], "dest": "login_no_browser", "nargs": 0, "const": true, "default": false, "help": "print the confirmation URL without opening a browser", "group": 1} + ] + } + }, + { + "name": "logout", + "parser": { + "prog": "sqb logout", + "description": "Revoke the saved CLI session and remove local credentials. Remote failures are reported, never silently treated as successful revocation.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin", "group": 1}, + {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1} + ] + } + }, { "name": "dev", "parser": { @@ -8701,6 +8729,8 @@ } ], "choices_help": [ + {"dest": "login", "metavar": "login", "help": "sign in through a browser or verify a saved CLI session"}, + {"dest": "logout", "metavar": "logout", "help": "revoke the CLI session and remove local credentials"}, { "dest": "dev", "metavar": "dev", diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index ff3bfcc8ea..689607a902 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -13,6 +13,7 @@ use crate::launcher::constants::{ use crate::native_commands::main::parse_native_invocation::parse_native_invocation; use crate::native_commands::main::run_control_invocation::run_control_invocation; use crate::native_commands::main::run_location_invocation::run_location_invocation; +use crate::native_commands::main::run_login_invocation::run_login_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; @@ -46,6 +47,11 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - if let Ok(NativeParse::Exit(code)) = &native { return *code; } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_login_invocation(namespace) + { + return code; + } if let Ok(NativeParse::Invocation { namespace, .. }) = &native && let Some(code) = run_control_invocation(namespace) { diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 5364b9c483..ce5297b160 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -9,5 +9,6 @@ pub mod diff_report; pub mod dispatch; pub mod launcher; pub mod native_commands; +pub(crate) mod person_login; pub mod replay; pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index f4a156f9bb..6ad71f45be 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -2,12 +2,12 @@ use serde_json::{Value, json}; use sqlbuild_orchestration::authorization::main::bootstrap_user::bootstrap_user; -use sqlbuild_orchestration::credentials::main::create_location::create_location; -use sqlbuild_orchestration::credentials::main::create_principal::create_principal; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; -use sqlbuild_orchestration::credentials::main::rotate_token::rotate_token; -use sqlbuild_orchestration::credentials::main::set_password::set_password; +use sqlbuild_orchestration::credentials::main::identities::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::locations::create_location::create_location; +use sqlbuild_orchestration::credentials::main::sessions::set_password::set_password; +use sqlbuild_orchestration::credentials::main::tokens::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::tokens::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::main::tokens::rotate_token::rotate_token; use sqlbuild_orchestration::credentials::models::{AuditContext, PasswordAuthority}; use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs index bbffbb5261..e852ca458c 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs @@ -3,7 +3,7 @@ use std::path::Path; use sqlbuild_orchestration::api::models::DevelopmentScope; -use sqlbuild_orchestration::credentials::main::development_location::development_location; +use sqlbuild_orchestration::credentials::main::locations::development_location::development_location; use sqlbuild_orchestration::credentials::models::AuditContext; use sqlbuild_orchestration::declarations::main::pause_development_triggers::pause_development_triggers; use sqlbuild_orchestration::store::errors::ControlError; diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index fab61b5d4b..18cbcdfe85 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -11,4 +11,5 @@ pub(super) mod _run_location; pub(crate) mod parse_native_invocation; pub(crate) mod run_control_invocation; pub(crate) mod run_location_invocation; +pub(crate) mod run_login_invocation; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs new file mode 100644 index 0000000000..62b3216e37 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs @@ -0,0 +1,48 @@ +//! Login resolves its native command options without project discovery or invocation capture. + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::person_login::main::run_login::run_login; +use crate::person_login::models::{LoginAction, LoginOptions}; + +pub(crate) fn run_login_invocation(namespace: &Namespace) -> Option { + let command = text(namespace, "command")?; + let action = match command.as_str() { + "logout" => LoginAction::Logout, + "login" if text(namespace, "login_action").as_deref() == Some("status") => { + LoginAction::Status + } + "login" => LoginAction::Start, + _ => return None, + }; + Some( + match run_login( + LoginOptions { + action, + server: text(namespace, "login_server").unwrap_or_default(), + ca: text(namespace, "login_ca").map(Into::into), + no_browser: flag(namespace, "login_no_browser"), + }, + |message| { + drop(write_output(&[(Stream::Stderr, message)])); + }, + ) { + Ok(Some(identity)) => match serde_json::to_string(&identity) { + Ok(identity) => { + let lost = write_output(&[(Stream::Stdout, &format!("{identity}\n"))]); + finish_lost_stdout(lost.as_ref(), 0) + } + Err(_) => 1, + }, + Ok(None) => 0, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("SQLBuild CLI {command} failed: {error}\n"), + )])); + 1 + } + }, + ) +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs b/crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs new file mode 100644 index 0000000000..f0a688d9c5 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs @@ -0,0 +1,200 @@ +//! Prefer the OS credential store; headless Unix fallback is private, owner-only and atomic. + +use crate::person_login::constants::KEYRING_SERVICE; +use crate::person_login::models::PersonCredential; + +fn headless_linux() -> bool { + cfg!(target_os = "linux") + && std::env::var_os("DBUS_SESSION_BUS_ADDRESS").is_none() + && std::env::var_os("XDG_RUNTIME_DIR").is_none() +} + +pub(crate) fn load(origin: &str) -> Result, std::io::Error> { + let encoded = match keyring::Entry::new(KEYRING_SERVICE, origin) + .and_then(|entry| entry.get_password()) + { + Ok(value) => Some(value), + Err(keyring::Error::NoEntry) => private_file(origin, None)?, + Err(keyring::Error::PlatformFailure(_)) if headless_linux() => private_file(origin, None)?, + Err(_) => { + return Err(std::io::Error::other( + "OS credential store could not be read; unlock it before retrying", + )); + } + }; + encoded + .map(|value| { + serde_json::from_str(&value).map_err(|_| { + std::io::Error::other("stored CLI credential is invalid; run sqb logout") + }) + }) + .transpose() +} + +pub(crate) fn save(origin: &str, credential: &PersonCredential) -> Result { + let encoded = serde_json::to_string(credential) + .map_err(|_| std::io::Error::other("could not encode the CLI credential"))?; + match keyring::Entry::new(KEYRING_SERVICE, origin) + .and_then(|entry| entry.set_password(&encoded)) + { + Ok(()) => { + remove_file(origin)?; + Ok(true) + } + Err(keyring::Error::PlatformFailure(_)) if headless_linux() => { + private_file(origin, Some(&encoded))?; + Ok(false) + } + Err(_) => Err(std::io::Error::other( + "OS credential store rejected persistence; unlock it before retrying", + )), + } +} + +pub(crate) fn remove(origin: &str) -> Result<(), std::io::Error> { + let removed = match keyring::Entry::new(KEYRING_SERVICE, origin) + .and_then(|entry| entry.delete_credential()) + { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(keyring::Error::PlatformFailure(_)) if headless_linux() => Ok(()), + Err(_) => Err(std::io::Error::other( + "OS credential store cleanup could not be confirmed", + )), + }; + remove_file(origin)?; + removed +} + +#[cfg(target_os = "linux")] +fn path(origin: &str, create: bool) -> Result, std::io::Error> { + use std::os::unix::fs::{DirBuilderExt, MetadataExt}; + let Some(home) = std::env::var_os("HOME").filter(|value| !value.is_empty()) else { + return if create { + Err(std::io::Error::other( + "a user home directory is required for credential storage", + )) + } else { + Ok(None) + }; + }; + let mut directory = std::path::PathBuf::from(home); + if !directory.is_absolute() { + return Err(std::io::Error::other( + "the credential home directory must be absolute", + )); + } + let credential = directory + .join(".config/sqlbuild/credentials") + .join(format!("{}.json", blake3::hash(origin.as_bytes()).to_hex())); + if !create { + match std::fs::symlink_metadata(&credential) { + Ok(_) => (), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + } + } + for (component, forbidden) in [ + ("", 0o022), + (".config", 0o022), + ("sqlbuild", 0o022), + ("credentials", 0o077), + ] { + if !component.is_empty() { + directory.push(component); + } + if create && !component.is_empty() { + match std::fs::DirBuilder::new().mode(0o700).create(&directory) { + Ok(()) => (), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => (), + Err(error) => return Err(error), + } + } + let metadata = std::fs::symlink_metadata(&directory)?; + if !metadata.is_dir() + || metadata.uid() != rustix::process::geteuid().as_raw() + || metadata.mode() & forbidden != 0 + { + return Err(std::io::Error::other( + "CLI credential directories must be owned by you, not symlinks or writable by others; the credential directory must be private", + )); + } + } + Ok(Some(credential)) +} + +#[cfg(target_os = "linux")] +fn private_file(origin: &str, encoded: Option<&str>) -> Result, std::io::Error> { + use std::io::{Read, Write}; + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let Some(path) = path(origin, encoded.is_some())? else { + return Ok(None); + }; + if let Some(encoded) = encoded { + let mut bytes = [0_u8; 16]; + getrandom::fill(&mut bytes).map_err(std::io::Error::other)?; + let temporary = path.with_extension(format!("{}.tmp", uuid::Uuid::from_bytes(bytes))); + let result = (|| { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(encoded.as_bytes())?; + file.sync_all()?; + std::fs::rename(&temporary, &path) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result?; + return Ok(None); + } + let mut file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32) + .open(&path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let metadata = file.metadata()?; + if !metadata.is_file() + || metadata.uid() != rustix::process::geteuid().as_raw() + || metadata.mode() & 0o077 != 0 + { + return Err(std::io::Error::other( + "CLI credential file must be owned by you and accessible only to you", + )); + } + let mut encoded = String::new(); + file.read_to_string(&mut encoded)?; + Ok(Some(encoded)) +} + +#[cfg(target_os = "linux")] +fn remove_file(origin: &str) -> Result { + let Some(path) = path(origin, false)? else { + return Ok(false); + }; + match std::fs::remove_file(path) { + Ok(()) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +#[cfg(not(target_os = "linux"))] +fn private_file(_origin: &str, encoded: Option<&str>) -> Result, std::io::Error> { + match encoded { + None => Ok(None), + Some(_) => Err(std::io::Error::other( + "an available OS credential store is required on this platform", + )), + } +} + +#[cfg(not(target_os = "linux"))] +fn remove_file(_origin: &str) -> Result { + Ok(false) +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs b/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs new file mode 100644 index 0000000000..3c14e27cb9 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs @@ -0,0 +1,117 @@ +//! A CLI-only random secret can be exchanged only after an explicit browser decision. + +use std::fmt::Write; +use std::time::{Duration, Instant}; + +use chrono::Utc; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::person_login::constants::{LOGIN_SECONDS, POLL_SECONDS, SECRET_BYTES}; +use crate::person_login::models::{LoginFlow, PersonClient, PersonCredential}; + +pub(crate) async fn login( + client: &PersonClient, + no_browser: bool, + report: &impl Fn(&str), +) -> Result { + let mut bytes = [0_u8; SECRET_BYTES]; + getrandom::fill(&mut bytes) + .map_err(|_| std::io::Error::other("could not generate a secure CLI exchange secret"))?; + let mut secret = String::new(); + for byte in bytes { + write!(secret, "{byte:02x}").map_err(std::io::Error::other)?; + } + let response = client + .client + .post(format!("{}/api/auth/cli/start", client.origin)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .map_err(|_| std::io::Error::other("could not start browser sign-in"))?; + if !response.status().is_success() { + return Err(std::io::Error::other( + "server rejected CLI sign-in; check its authentication configuration or wait before retrying", + )); + } + let flow: LoginFlow = response + .json() + .await + .map_err(|_| std::io::Error::other("server returned an invalid CLI handoff"))?; + let (id, code) = flow + .browser_code + .split_once('.') + .ok_or_else(|| std::io::Error::other("server returned an invalid CLI handoff"))?; + let id = uuid::Uuid::parse_str(id) + .map_err(|_| std::io::Error::other("server returned an invalid CLI handoff"))?; + if code.len() != SECRET_BYTES * 2 || !code.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(std::io::Error::other( + "server returned an invalid CLI handoff", + )); + } + let url = format!("{}/auth/cli#{}", client.origin, flow.browser_code); + report(&format!( + "Open {url}\nVerification code: {}\nWaiting for browser authorization…\n", + code[..8].to_ascii_uppercase() + )); + if !no_browser && !open_browser(&url) { + report("Could not open a browser automatically. Open the URL above manually.\n"); + } + let started = Instant::now(); + while started.elapsed() < Duration::from_secs(LOGIN_SECONDS) && Utc::now() < flow.expires_at { + let response = client + .client + .post(format!("{}/api/auth/cli/claim", client.origin)) + .json(&json!({"flow_id":id,"client_secret":secret})) + .send() + .await + .map_err(|_| { + std::io::Error::other("could not complete CLI sign-in; start sqb login again") + })?; + match response.status() { + reqwest::StatusCode::ACCEPTED => { + tokio::time::sleep(Duration::from_secs(POLL_SECONDS)).await + } + reqwest::StatusCode::OK => { + return response.json().await.map_err(|_| { + std::io::Error::other("server returned an invalid CLI credential") + }); + } + _ => { + return Err(std::io::Error::other( + "CLI sign-in was cancelled, expired or is unavailable; start sqb login again", + )); + } + } + } + Err(std::io::Error::other( + "CLI sign-in expired; start sqb login again", + )) +} + +fn open_browser(url: &str) -> bool { + #[cfg(target_os = "macos")] + let mut command = std::process::Command::new("open"); + #[cfg(all(unix, not(target_os = "macos")))] + let mut command = std::process::Command::new("xdg-open"); + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("rundll32"); + command.arg("url.dll,FileProtocolHandler"); + command + }; + command + .arg(url) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + match command.spawn() { + Ok(mut child) => { + std::thread::spawn(move || { + let _ = child.wait(); + }); + true + } + Err(_) => false, + } +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs new file mode 100644 index 0000000000..11f145944a --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(super) mod credentials; +pub(super) mod flow; +pub(super) mod transport; diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs new file mode 100644 index 0000000000..b62d3e990c --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs @@ -0,0 +1,101 @@ +//! HTTPS identity is verified; redirects cannot forward person credentials to another origin. + +use std::path::Path; +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderValue}; +use sqlbuild_orchestration::credentials::models::SessionIdentity; + +use crate::person_login::constants::REQUEST_SECONDS; +use crate::person_login::models::{PersonClient, PersonCredential}; + +pub(crate) fn connect(origin: &str, ca: Option<&Path>) -> Result { + let mut builder = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(REQUEST_SECONDS)) + .default_headers( + [( + reqwest::header::ORIGIN, + HeaderValue::from_str(origin).map_err(std::io::Error::other)?, + )] + .into_iter() + .collect(), + ); + if let Some(ca) = ca { + builder = builder.add_root_certificate( + reqwest::Certificate::from_pem(&std::fs::read(ca)?) + .map_err(|_| std::io::Error::other("could not read the trusted CA certificate"))?, + ); + } + Ok(PersonClient { + client: builder + .build() + .map_err(|_| std::io::Error::other("could not initialize the HTTPS client"))?, + origin: origin.into(), + }) +} + +pub(crate) fn authorized( + client: &PersonClient, + method: reqwest::Method, + path: &str, + credential: &PersonCredential, +) -> Result { + let mut authorization = HeaderValue::from_str(&format!("Session {}", credential.bearer)) + .map_err(|_| std::io::Error::other("stored CLI credential is invalid"))?; + authorization.set_sensitive(true); + let mut csrf = HeaderValue::from_str(&credential.csrf_token) + .map_err(|_| std::io::Error::other("stored CLI credential is invalid"))?; + csrf.set_sensitive(true); + Ok(client + .client + .request(method, format!("{}{path}", client.origin)) + .header(AUTHORIZATION, authorization) + .header("x-sqb-csrf", csrf)) +} + +pub(crate) async fn current( + client: &PersonClient, + credential: &PersonCredential, +) -> Result { + let response = authorized( + client, + reqwest::Method::GET, + "/api/auth/session", + credential, + )? + .send() + .await + .map_err(|_| std::io::Error::other("could not verify the CLI session with the server"))?; + if !response.status().is_success() { + return Err(std::io::Error::other( + "CLI session is unavailable or revoked; run sqb login again", + )); + } + response + .json() + .await + .map_err(|_| std::io::Error::other("server returned an invalid person identity")) +} + +pub(crate) async fn revoke( + client: &PersonClient, + credential: &PersonCredential, +) -> Result<(), std::io::Error> { + let response = authorized( + client, + reqwest::Method::POST, + "/api/auth/logout", + credential, + )? + .send() + .await + .map_err(|_| std::io::Error::other("remote session revocation could not be confirmed"))?; + if response.status().is_success() || response.status() == reqwest::StatusCode::UNAUTHORIZED { + Ok(()) + } else { + Err(std::io::Error::other( + "remote session revocation could not be confirmed", + )) + } +} diff --git a/crates/sqlbuild-cli/src/person_login/constants.rs b/crates/sqlbuild-cli/src/person_login/constants.rs new file mode 100644 index 0000000000..d525dc760e --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/constants.rs @@ -0,0 +1,7 @@ +//! Person login bounds and user-owned credential storage names. + +pub(super) const SECRET_BYTES: usize = 32; +pub(super) const POLL_SECONDS: u64 = 1; +pub(super) const LOGIN_SECONDS: u64 = 300; +pub(super) const REQUEST_SECONDS: u64 = 15; +pub(super) const KEYRING_SERVICE: &str = "sqlbuild-cli"; diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs new file mode 100644 index 0000000000..b64986bbb9 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod run_login; diff --git a/crates/sqlbuild-cli/src/person_login/main/run_login.rs b/crates/sqlbuild-cli/src/person_login/main/run_login.rs new file mode 100644 index 0000000000..08fb898dc2 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/run_login.rs @@ -0,0 +1,70 @@ +//! Person login commands run before project discovery and never write credentials into a project. + +use crate::person_login::_helpers::{credentials, flow, transport}; +use crate::person_login::models::{LoginAction, LoginOptions}; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; +use sqlbuild_orchestration::credentials::models::SessionIdentity; + +pub(crate) fn run_login( + options: LoginOptions, + report: impl Fn(&str), +) -> Result, std::io::Error> { + let origin = parse_origin(&options.server, "CLI").map_err(std::io::Error::other)?; + let (logout_credential, cleanup) = if options.action == LoginAction::Logout { + report("Signing out of SQLBuild CLI…\n"); + let credential = credentials::load(&origin); + let cleanup = credentials::remove(&origin); + match credential { + Ok(Some(credential)) => (Some(credential), cleanup), + Ok(None) => { + cleanup?; + report("SQLBuild CLI signed out successfully\n"); + return Ok(None); + } + Err(error) => { + cleanup?; + return Err(error); + } + } + } else { + (None, Ok(())) + }; + let client = transport::connect(&origin, options.ca.as_deref())?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(async { + if let Some(credential) = logout_credential { + let remote = transport::revoke(&client, &credential).await; + cleanup?; + remote?; + report("SQLBuild CLI signed out successfully\n"); + return Ok(None); + } + if options.action == LoginAction::Status { + report("Checking SQLBuild CLI session…\n"); + let credential = credentials::load(&client.origin)? + .ok_or_else(|| std::io::Error::other("not signed in; run sqb login"))?; + let identity = transport::current(&client, &credential).await?; + report("SQLBuild CLI session verified successfully\n"); + return Ok(Some(identity)); + } + report("Starting SQLBuild CLI sign-in…\n"); + let credential = flow::login(&client, options.no_browser, &report).await?; + match credentials::save(&client.origin, &credential) { + Ok(true) => (), + Ok(false) => report("OS credential store unavailable; using an owner-only credential file in your user configuration directory.\n"), + Err(error) => { + let _ = transport::revoke(&client, &credential).await; + return Err(error); + } + } + if let Err(error) = transport::current(&client, &credential).await { + let _ = transport::revoke(&client, &credential).await; + credentials::remove(&client.origin)?; + return Err(error); + } + report("SQLBuild CLI signed in successfully\n"); + Ok::<_, std::io::Error>(None) + }) +} diff --git a/crates/sqlbuild-cli/src/person_login/mod.rs b/crates/sqlbuild-cli/src/person_login/mod.rs new file mode 100644 index 0000000000..1f16a57e51 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/mod.rs @@ -0,0 +1,6 @@ +//! Browser-authorized person sessions are distinct from service-account and worker tokens. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/person_login/models.rs b/crates/sqlbuild-cli/src/person_login/models.rs new file mode 100644 index 0000000000..e46af04fe4 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/models.rs @@ -0,0 +1,58 @@ +//! Only public identity is printable; persisted session material is always redacted. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlbuild_orchestration::credentials::models::SessionIdentity; + +#[derive(Debug)] +pub(crate) struct LoginOptions { + pub(crate) action: LoginAction, + pub(crate) server: String, + pub(crate) ca: Option, + pub(crate) no_browser: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LoginAction { + Start, + Status, + Logout, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PersonCredential { + pub(super) identity: SessionIdentity, + pub(super) bearer: String, + pub(super) csrf_token: String, +} + +impl std::fmt::Debug for PersonCredential { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PersonCredential") + .field("identity", &self.identity) + .finish_non_exhaustive() + } +} + +#[derive(Deserialize)] +pub(super) struct LoginFlow { + pub(super) browser_code: String, + pub(super) expires_at: DateTime, +} + +impl std::fmt::Debug for LoginFlow { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("LoginFlow") + .field("expires_at", &self.expires_at) + .finish_non_exhaustive() + } +} + +#[derive(Debug)] +pub(super) struct PersonClient { + pub(super) client: reqwest::Client, + pub(super) origin: String, +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index 6cbfc0b2d2..7c7dd5635f 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -5,7 +5,7 @@ use std::process::{ExitStatus, Stdio}; use std::time::{Duration, Instant}; use chrono::{TimeDelta, Utc}; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::tokens::issue_token::issue_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; use sqlbuild_orchestration::runs::main::advance_groups::advance_groups; use sqlbuild_orchestration::runs::main::change_run::change_run; diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index a615b93b6e..7f1a901fd6 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -4,9 +4,9 @@ use std::collections::BTreeMap; use std::process::{Command, Stdio}; use std::time::{Duration, SystemTime}; -use sqlbuild_orchestration::credentials::main::create_location::create_location; -use sqlbuild_orchestration::credentials::main::issue_token::issue_token; -use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::main::locations::create_location::create_location; +use sqlbuild_orchestration::credentials::main::tokens::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::tokens::revoke_token::revoke_token; use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, IssuedToken}; use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs index fdf2cdb3ec..014cc2ec71 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs @@ -10,7 +10,7 @@ use uuid::Uuid; use crate::api::errors::ApiError; use crate::api::models::{DevelopmentScope, ReadResource}; -use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::main::tokens::authenticate_token::authenticate_token; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html new file mode 100644 index 0000000000..8880c22476 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html @@ -0,0 +1,48 @@ + + + + + +SQLBuild CLI sign-in + + + + +
+

Sign in to SQLBuild CLI

+

Only continue if you started sqb login yourself. Compare this code with your terminal:

+— +

Checking your sign-in…

+ + + + + +
+ + diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js new file mode 100644 index 0000000000..c0f479c543 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js @@ -0,0 +1,78 @@ +"use strict"; + +window.addEventListener("hashchange", () => location.reload()); +const element = (id) => document.getElementById(id); +const storageKey = "sqb.cli.handoff"; +const code = location.hash.slice(1) || sessionStorage.getItem(storageKey) || ""; +history.replaceState(null, "", location.pathname); +const valid = /^[0-9a-f-]{36}\.[0-9a-f]{64}$/i.test(code); +if (valid) sessionStorage.setItem(storageKey, code); +const message = (text) => { element("message").textContent = text; }; +const csrf = () => document.cookie.split(";").map((part) => part.trim()).find((part) => part.startsWith("__Host-sqb-csrf="))?.split("=")[1] || ""; + +async function request(path, body) { + return fetch(path, { method: "POST", credentials: "same-origin", headers: { "Content-Type": "application/json", "x-sqb-csrf": csrf() }, body: JSON.stringify(body) }); +} + +async function current() { + const response = await fetch("/api/auth/session", { credentials: "same-origin" }); + if (!response.ok) { + message("Sign in before authorizing your CLI."); + element(document.body.dataset.method === "password" ? "password" : document.body.dataset.method === "oidc" ? "sso" : "proxy").hidden = false; + return; + } + const identity = await response.json(); + for (const id of ["password", "sso", "proxy"]) element(id).hidden = true; + element("account").textContent = `Signed in as person ${identity.principal_id}`; + element("confirmation").hidden = false; + message("Check that the verification code matches your terminal before authorizing."); +} + +element("password").addEventListener("submit", async (event) => { + event.preventDefault(); + const button = element("password").querySelector("button"); + button.disabled = true; + try { + const response = await request("/api/auth/password/login", { username: element("username").value, password: element("password-value").value }); + if (!response.ok) { message("Sign-in was rejected. Check your credentials, or wait before trying again."); return; } + await current(); + } catch { message("Could not reach SQLBuild. Try signing in again."); } + finally { element("password-value").value = ""; button.disabled = false; } +}); + +element("sso").addEventListener("click", () => { + document.cookie = "__Host-sqb-cli-return=1; Secure; SameSite=Lax; Path=/; Max-Age=300"; + location.assign("/api/auth/oidc/login"); +}); + +element("proxy").addEventListener("click", async () => { + element("proxy").disabled = true; + try { + const response = await fetch("/api/auth/proxy", { credentials: "same-origin" }); + if (!response.ok) { message("Organization sign-in was rejected. Contact your administrator."); return; } + await current(); + } catch { message("Could not reach SQLBuild. Try again."); } + finally { element("proxy").disabled = false; } +}); + +async function decide(approve) { + element("approve").disabled = true; + element("cancel").disabled = true; + try { + const response = await request("/api/auth/cli/approve", { browser_code: code, approve }); + if (!response.ok) { message("This sign-in is no longer available. Start sqb login again."); return; } + sessionStorage.removeItem(storageKey); + element("confirmation").hidden = true; + message(approve ? "CLI authorized. Return to your terminal to finish signing in." : "CLI sign-in cancelled. No CLI credential was issued."); + } catch { message("Could not confirm the decision. Start sqb login again."); } +} +element("approve").addEventListener("click", () => decide(true)); +element("cancel").addEventListener("click", () => decide(false)); + +if (!valid) { + sessionStorage.removeItem(storageKey); + message("Start sqb login in your terminal to open a valid sign-in request."); +} else { + element("code").textContent = code.split(".")[1].slice(0, 8).toUpperCase(); + current().catch(() => message("Could not reach SQLBuild. Start sqb login again.")); +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs new file mode 100644 index 0000000000..84690b2739 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs @@ -0,0 +1,122 @@ +//! Only the exchange secret can receive a CLI credential; browser approval returns no bearer. + +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{Html, IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Extension, Json, Router}; + +use crate::api::_helpers::authentication::password_transport::{ + browser_cookie, deny, single_header, +}; +use crate::api::constants::{PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_HEADER}; +use crate::api::errors::ApiError; +use crate::api::models::{ + AuthenticationDenial, BrowserLoginMethod, CliLoginApprovalRequest, CliLoginClaimRequest, + CliLoginSessionResponse, CliLoginStartRequest, +}; +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::cli_login::{ + approve_login::approve_login, claim_login::claim_login, start_login::start_login, +}; +use crate::credentials::models::{AuditContext, AuthenticationAttempt, ClaimedLogin, StartedLogin}; +use crate::store::models::ControlStore; + +pub(crate) fn router(method: BrowserLoginMethod) -> Router { + Router::new() + .route("/auth/cli", get(page)) + .route("/auth/cli.js", get(script)) + .route("/api/auth/cli/start", post(start)) + .route("/api/auth/cli/claim", post(claim)) + .layer(Extension(method)) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) +} + +async fn page(Extension(method): Extension) -> Response { + let method = match method { + BrowserLoginMethod::Password => "password", + BrowserLoginMethod::Oidc => "oidc", + BrowserLoginMethod::Proxy => "proxy", + }; + let mut response = + Html(include_str!("cli_login.html").replace("__LOGIN_METHOD__", method)).into_response(); + response.headers_mut().insert(header::CONTENT_SECURITY_POLICY, + header::HeaderValue::from_static("default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'; form-action 'self'; base-uri 'none'")); + response.headers_mut().insert( + header::REFERRER_POLICY, + header::HeaderValue::from_static("no-referrer"), + ); + response +} + +async fn script() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "text/javascript; charset=utf-8")], + include_str!("cli_login.js"), + ) +} + +#[utoipa::path(post, path = "/api/auth/cli/start", operation_id = "startCliLogin", + request_body = CliLoginStartRequest, responses((status = 200, body = StartedLogin), + (status = 429, body = AuthenticationDenial)))] +pub(crate) async fn start( + State(store): State, + Extension(context): Extension, + Json(request): Json, +) -> Result { + let (_, allowed) = + reserve_authentication_attempt(&store, &context, AuthenticationAttempt::IdentityProvider) + .await?; + if !allowed { + let mut response = deny(&store, &context, "cli_login_throttled").await?; + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + return Ok(response); + } + Ok(Json(start_login(&store, &context, &request.client_hash).await?).into_response()) +} + +#[utoipa::path(post, path = "/api/auth/cli/approve", operation_id = "approveCliLogin", + request_body = CliLoginApprovalRequest, security(("session" = [])), + responses((status = 204), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn approve( + State(store): State, + Extension(context): Extension, + headers: HeaderMap, + Json(request): Json, +) -> Result { + if approve_login( + &store, + &context, + ( + browser_cookie(&headers, PASSWORD_COOKIE).unwrap_or_default(), + single_header(&headers, PASSWORD_CSRF_HEADER).unwrap_or_default(), + ), + (&request.browser_code, request.approve), + ) + .await? + { + Ok(StatusCode::NO_CONTENT.into_response()) + } else { + deny(&store, &context, "invalid_cli_approval").await + } +} + +#[utoipa::path(post, path = "/api/auth/cli/claim", operation_id = "claimCliLogin", + request_body = CliLoginClaimRequest, responses((status = 200, body = CliLoginSessionResponse), + (status = 202), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn claim( + State(store): State, + Extension(context): Extension, + Json(request): Json, +) -> Result { + match claim_login(&store, &context, (request.flow_id, &request.client_secret)).await? { + ClaimedLogin::Pending => Ok(StatusCode::ACCEPTED.into_response()), + ClaimedLogin::Unavailable => deny(&store, &context, "cli_login_unavailable").await, + ClaimedLogin::Authenticated(session) => Ok(Json(CliLoginSessionResponse { + identity: session.identity, + bearer: session.bearer, + csrf_token: session.csrf, + }) + .into_response()), + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs index a8a19eeee6..95a52ea8b5 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod bearer; +pub(crate) mod cli_login; pub(crate) mod configure; pub(crate) mod oidc; pub(crate) mod oidc_provider; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs index c17062cb61..95a634c4e9 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse, Redirect, Response}; use axum::routing::get; use axum::{Extension, Router, middleware}; @@ -17,13 +17,13 @@ use crate::api::_helpers::authentication::{oidc_provider, password_sessions}; use crate::api::constants::OIDC_FLOW_COOKIE; use crate::api::errors::ApiError; use crate::api::models::{ - AuthenticationDenial, BrowserOrigin, ExternalIdentityResponse, OidcAuthentication, - OidcCallback, OidcRuntime, + AuthenticationDenial, BrowserLoginMethod, BrowserOrigin, ExternalIdentityResponse, + OidcAuthentication, OidcCallback, OidcRuntime, }; use crate::credentials::constants::OIDC_FLOW_SECONDS; -use crate::credentials::main::authenticate_bound_person::authenticate_bound_person; -use crate::credentials::main::reserve_authentication_attempt::reserve_authentication_attempt; -use crate::credentials::main::{ +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::sessions::authenticate_bound_person::authenticate_bound_person; +use crate::credentials::main::sessions::{ claim_oidc_flow::claim_oidc_flow, create_oidc_flow::create_oidc_flow, }; use crate::credentials::models::{ @@ -45,6 +45,7 @@ pub(crate) async fn router( .route("/api/auth/oidc/login", get(login)) .route("/api/auth/oidc/callback", get(callback)) .merge(password_sessions::protected(store)) + .merge(super::cli_login::router(BrowserLoginMethod::Oidc)) .layer(middleware::from_fn_with_state(store.clone(), guard)) .layer(Extension(BrowserOrigin(runtime.settings.origin.clone()))) .layer(Extension(Arc::new(runtime)))) @@ -147,6 +148,22 @@ pub(crate) async fn callback( }, Err(_) => deny(&store, &context, "invalid_oidc_identity").await?, }; + if browser_cookie(&headers, "__Host-sqb-cli-return") == Some("1") { + set_cookie( + &mut response, + ("__Host-sqb-cli-return", ""), + 0, + (false, "Lax"), + )?; + if response.status().is_success() { + *response.status_mut() = StatusCode::SEE_OTHER; + response.headers_mut().insert( + header::LOCATION, + header::HeaderValue::from_static("/auth/cli"), + ); + *response.body_mut() = axum::body::Body::empty(); + } + } set_cookie(&mut response, (OIDC_FLOW_COOKIE, ""), 0, (true, "Lax"))?; Ok(response) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs index 2976cf2ba5..4c71c7f501 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs @@ -9,15 +9,15 @@ use openidconnect::{ }; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use crate::api::_helpers::authentication::password_transport::parse_origin; use crate::api::constants::{HTTPS_SCHEME, OIDC_RESPONSE_BYTES}; +use crate::api::main::parse_origin::parse_origin; use crate::api::models::{OidcAuthentication, OidcRuntime}; use crate::api::types::OidcClient; use crate::credentials::models::StoredOidcFlow; use crate::store::errors::ControlError; pub(crate) fn configure(mut settings: OidcAuthentication) -> Result { - settings.origin = parse_origin(&settings.origin, "OIDC")?.0; + settings.origin = parse_origin(&settings.origin, "OIDC")?; let issuer = https_url(&settings.issuer)?; if issuer.query().is_some() || issuer.fragment().is_some() { return Err(ControlError::Invalid( diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 2f11dced21..ce24edb6c1 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -7,17 +7,20 @@ use axum::routing::{get, post}; use axum::{Extension, Json, Router, middleware}; use crate::api::_helpers::authentication::password_transport::{ - authenticate, deny, guard, parse_origin, session_cookies, set_cookie, + authenticate, deny, guard, session_cookies, set_cookie, }; use crate::api::constants::{PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE}; use crate::api::errors::ApiError; +use crate::api::main::parse_origin::parse_origin; use crate::api::models::{ - AuthenticationDenial, PasswordChangeRequest, PasswordLoginRequest, PasswordSessionResponse, + AuthenticationDenial, BrowserLoginMethod, BrowserOrigin, PasswordChangeRequest, + PasswordLoginRequest, PasswordSessionResponse, }; use crate::credentials::constants::LOGIN_WINDOW_SECONDS; use crate::credentials::main::{ - login_password::login_password, reserve_authentication_attempt::reserve_authentication_attempt, - revoke_session::revoke_session, set_password::set_password, verify_password::verify_password, + audit::reserve_authentication_attempt::reserve_authentication_attempt, + sessions::login_password::login_password, sessions::revoke_session::revoke_session, + sessions::set_password::set_password, sessions::verify_password::verify_password, }; use crate::credentials::models::{ AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, SessionIdentity, @@ -30,6 +33,7 @@ pub(crate) fn protected(store: &ControlStore) -> Router { Router::new() .route("/api/auth/session", get(current)) .route("/api/auth/logout", post(logout)) + .route("/api/auth/cli/approve", post(super::cli_login::approve)) .route( "/api/control/audit", get(crate::api::_helpers::resources::audit::read), @@ -97,7 +101,7 @@ pub(crate) fn router( let Some(origin) = origin else { return Ok(Router::new()); }; - let origin = parse_origin(&origin, "password")?; + let origin = BrowserOrigin(parse_origin(&origin, "password")?); let protected = protected(store).merge( Router::new() .route("/api/auth/password", post(change_password)) @@ -105,6 +109,7 @@ pub(crate) fn router( ); Ok(protected .route("/api/auth/password/login", post(login)) + .merge(super::cli_login::router(BrowserLoginMethod::Password)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) .layer(middleware::from_fn_with_state(store.clone(), guard)) .layer(Extension(origin))) @@ -159,7 +164,7 @@ pub(crate) async fn login( #[utoipa::path(get, path = "/api/auth/session", operation_id = "currentSession", responses((status = 200, body = SessionIdentity), (status = 401, body = AuthenticationDenial)), - security(("session" = [])))] + security(("session" = []), ("cliSession" = [])))] pub(crate) async fn current( Extension(identity): Extension, ) -> Json { @@ -169,7 +174,7 @@ pub(crate) async fn current( #[utoipa::path(post, path = "/api/auth/logout", operation_id = "logoutSession", params(("x-sqb-csrf" = String, Header, description = "Session CSRF token")), responses((status = 204), (status = 401, body = AuthenticationDenial)), - security(("session" = [])))] + security(("session" = []), ("cliSession" = [])))] pub(crate) async fn logout( State(store): State, Extension(context): Extension, @@ -189,7 +194,7 @@ pub(crate) async fn logout( params(("x-sqb-csrf" = String, Header, description = "Session CSRF token")), request_body = PasswordChangeRequest, responses((status = 204), (status = 401, body = AuthenticationDenial)), - security(("session" = [])))] + security(("session" = []), ("cliSession" = [])))] pub(crate) async fn change_password( State(store): State, Extension(context): Extension, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs index cd21401204..792c4cd2a0 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -10,42 +10,16 @@ use axum::{Extension, Json}; use serde_json::json; use uuid::Uuid; -use crate::api::constants::{ - HTTPS_SCHEME, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE, PASSWORD_CSRF_HEADER, ROOT_PATH, -}; +use crate::api::constants::{PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE, PASSWORD_CSRF_HEADER}; use crate::api::errors::ApiError; use crate::api::models::{AuthenticationDenial, BrowserOrigin, ExternalIdentityResponse}; use crate::credentials::constants::PASSWORD_SESSION_SECONDS; -use crate::credentials::main::authenticate_session::authenticate_session; -use crate::credentials::main::record_authentication_denial::record_authentication_denial; +use crate::credentials::main::audit::record_authentication_denial::record_authentication_denial; +use crate::credentials::main::sessions::authenticate_session::authenticate_session; use crate::credentials::models::{AuditContext, IssuedSession, SessionAccess}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; -pub(crate) fn parse_origin(value: &str, mechanism: &str) -> Result { - let error = || { - ControlError::Invalid(format!( - "{mechanism} authentication requires an HTTPS origin without credentials, path, query or fragment" - )) - }; - let url = reqwest::Url::parse(value).map_err(|_| { - ControlError::Invalid(format!( - "{mechanism} authentication requires an HTTPS origin" - )) - })?; - if url.scheme() != HTTPS_SCHEME - || url.host().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.path() != ROOT_PATH - || url.query().is_some() - || url.fragment().is_some() - { - return Err(error()); - } - Ok(BrowserOrigin(url.origin().ascii_serialization())) -} - pub(crate) async fn guard( State(store): State, Extension(origin): Extension, @@ -110,12 +84,8 @@ pub(crate) async fn authenticate( csrf: single_header(request.headers(), PASSWORD_CSRF_HEADER).unwrap_or_default(), } }; - let identity = authenticate_session( - &store, - browser_cookie(request.headers(), PASSWORD_COOKIE).unwrap_or_default(), - access, - ) - .await?; + let identity = + authenticate_session(&store, session_credentials(request.headers()).0, access).await?; let Some(identity) = identity else { return deny(&store, &context, "invalid_session_or_csrf").await; }; @@ -136,8 +106,18 @@ pub(crate) fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&' } pub(crate) fn session_credentials(headers: &HeaderMap) -> (&str, &str) { + let cookie = browser_cookie(headers, PASSWORD_COOKIE); + let authorization = if headers.contains_key(header::AUTHORIZATION) { + if headers.contains_key(header::COOKIE) { + return ("", ""); + } + single_header(headers, header::AUTHORIZATION.as_str()) + .and_then(|value| value.strip_prefix("Session ")) + } else { + cookie + }; ( - browser_cookie(headers, PASSWORD_COOKIE).unwrap_or_default(), + authorization.unwrap_or_default(), single_header(headers, PASSWORD_CSRF_HEADER).unwrap_or_default(), ) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs index a1fcfc2c99..94c8788154 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs @@ -4,15 +4,17 @@ use std::sync::Arc; use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::password_transport::{ - deny, external_session, guard, parse_origin, single_header, + deny, external_session, guard, single_header, }; use crate::api::constants::PROXY_USER_HEADER; use crate::api::errors::ApiError; +use crate::api::main::parse_origin::parse_origin; use crate::api::models::{ - AuthenticationDenial, ExternalIdentityResponse, TrustedProxyAuthentication, + AuthenticationDenial, BrowserLoginMethod, BrowserOrigin, ExternalIdentityResponse, + TrustedProxyAuthentication, }; -use crate::credentials::main::authenticate_bound_person::authenticate_bound_person; -use crate::credentials::main::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::sessions::authenticate_bound_person::authenticate_bound_person; use crate::credentials::models::{AuditContext, AuthenticationAttempt, ExternalIdentityMechanism}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -29,7 +31,7 @@ pub(crate) fn router( let Some(mut settings) = settings else { return Ok(Router::new()); }; - let origin = parse_origin(&settings.origin, "trusted-proxy")?; + let origin = BrowserOrigin(parse_origin(&settings.origin, "trusted-proxy")?); if settings.trusted_peers.is_empty() { return Err(ControlError::Invalid( "trusted-proxy authentication requires explicit trusted peer IP addresses".into(), @@ -55,6 +57,7 @@ pub(crate) fn router( Ok(Router::new() .route("/api/auth/proxy", get(identity)) .merge(password_sessions::protected(store)) + .merge(super::cli_login::router(BrowserLoginMethod::Proxy)) .layer(middleware::from_fn_with_state(store.clone(), guard)) .layer(Extension(origin)) .layer(Extension(Arc::new(settings)))) diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs index 0fcc2fae9a..60405dbf31 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -19,7 +19,7 @@ use crate::store::models::ControlStore; params(("principal_id" = Uuid, Path), ("x-sqb-csrf" = String, Header, description = "Session CSRF token")), request_body = AdministratorChange, responses((status = 200, body = MutationResponse), (status = 401, body = crate::api::models::AuthenticationDenial), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn change( (State(store), Extension(context), Extension(authentication)): ( State, @@ -44,7 +44,7 @@ pub(crate) async fn change( #[utoipa::path(post, path = "/api/admin/roles", operation_id = "assignBuiltinRole", params(("x-sqb-csrf" = String, Header)), request_body = RoleAssignmentRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn assign( (State(store), Extension(context)): (State, Extension), Extension(authentication): Extension, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs index 4972c6f79c..2a39d21a79 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs @@ -15,7 +15,7 @@ use crate::store::models::ControlStore; params(("after" = Option, Query, description = "Exclusive storage-order cursor; pre-upgrade facts are ordered by timestamp and event ID"), ("limit" = Option, Query)), responses((status = 200, body = Vec), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), - security(("session" = [])))] + security(("session" = []), ("cliSession" = [])))] pub(crate) async fn read( (State(store), Extension(context)): (State, Extension), Query(options): Query, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs index 70e341de64..6771cd3ca5 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs @@ -15,7 +15,7 @@ use uuid::Uuid; use crate::api::_helpers::authentication::bearer::{bearer, require_location}; use crate::api::errors::ApiError; use crate::api::models::{ChangeFeed, ChangeNotice, ChangeStream}; -use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::main::tokens::authenticate_token::authenticate_token; use crate::credentials::models::{CredentialScope, TokenIdentity}; use crate::store::errors::ControlError; use crate::store::main::listen_changes::listen_changes; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs index a18cb71386..f54a60cb8b 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs @@ -22,7 +22,7 @@ use crate::store::models::ControlStore; request_body = ConnectionTestRequest, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), - security(("session" = [])))] + security(("session" = []), ("cliSession" = [])))] pub(crate) async fn request( (State(store), Extension(context)): (State, Extension), Path((location_id, target)): Path<(Uuid, String)>, @@ -50,7 +50,7 @@ pub(crate) async fn request( params(("location_id" = Uuid, Path), ("target" = String, Path), ("request_id" = Uuid, Path), ("x-sqb-csrf" = String, Header)), responses((status = 200, body = Option), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn result( (State(store), Extension(context)): (State, Extension), Path((location_id, target, request_id)): Path<(Uuid, String, Uuid)>, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index aca4c6b73c..e7c8adf48d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -1,4 +1,4 @@ -//! Person operations use browser authentication, never the machine-resource bearer router. +//! Person operations use person sessions, never the machine-resource bearer router. use axum::extract::{Path, Query, State}; use axum::http::HeaderMap; @@ -34,7 +34,7 @@ use crate::warehouse_metadata::models::{ ("after_event_id" = Option, Query), ("limit" = Option, Query)), responses((status = 200, body = StatisticsPage), (status = 400, description = "Invalid relation or cursor"), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn statistics( (State(store), Extension(context)): (State, Extension), Extension(default_role): Extension, @@ -55,7 +55,7 @@ pub(crate) async fn statistics( #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn request_job( (State(store), Extension(context)): (State, Extension), headers: HeaderMap, @@ -74,7 +74,7 @@ pub(crate) async fn request_job( #[utoipa::path(post, path = "/api/control/runs/cancel", operation_id = "cancelPersonRun", params(("x-sqb-csrf" = String, Header)), request_body = CancelRunRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn cancel_run( (State(store), Extension(context)): (State, Extension), headers: HeaderMap, @@ -89,7 +89,7 @@ pub(crate) async fn cancel_run( #[utoipa::path(post, path = "/api/control/runs/rerun", operation_id = "rerunPersonRun", params(("x-sqb-csrf" = String, Header)), request_body = RerunRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn rerun( (State(store), Extension(context)): (State, Extension), headers: HeaderMap, @@ -108,7 +108,7 @@ pub(crate) async fn rerun( #[utoipa::path(post, path = "/api/control/triggers", operation_id = "changeTrigger", params(("x-sqb-csrf" = String, Header)), request_body = TriggerChangeRequest, responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn change_trigger( (State(store), Extension(context)): (State, Extension), headers: HeaderMap, @@ -123,7 +123,7 @@ pub(crate) async fn change_trigger( #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs", operation_id = "listPersonRuns", params(("location_id" = Uuid, Path), ("target" = String, Path), ("limit" = Option, Query)), responses((status = 200, body = Vec), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn recent_runs( (State(store), Extension(context)): (State, Extension), Extension(default_role): Extension, @@ -150,7 +150,7 @@ pub(crate) async fn recent_runs( ("limit" = Option, Query, description = "Page size; defaults to 100, clamped to 1..100")), responses((status = 200, body = CatalogPage), (status = 400, description = "Invalid namespace or cursor"), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn catalog( (State(store), Extension(context)): (State, Extension), Extension(default_role): Extension, @@ -176,7 +176,7 @@ pub(crate) async fn catalog( ("name" = Option, Query, description = "Exact destination name; omitted matches SQL NULL")), responses((status = 200, body = Option), (status = 400, description = "Invalid node"), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn fingerprint( (State(store), Extension(context)): (State, Extension), Extension(default_role): Extension, @@ -199,7 +199,7 @@ pub(crate) async fn fingerprint( ("node_key" = String, Query, description = "One exact published node key")), responses((status = 200, body = Option), (status = 400, description = "Invalid node key"), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = [])))] + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn node_status( (State(store), Extension(context)): (State, Extension), Extension(default_role): Extension, diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs index f445706317..db4d8c57d7 100644 --- a/crates/sqlbuild-orchestration/src/api/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod client_response; pub(crate) mod connect_client; pub mod openapi_document; +pub mod parse_origin; pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs index e7a46a5b8e..21bea0101a 100644 --- a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -23,6 +23,16 @@ pub fn openapi_document() -> utoipa::openapi::OpenApi { "session", SecurityScheme::ApiKey(ApiKey::Cookie(ApiKeyValue::new(PASSWORD_COOKIE))), ); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "cliSession", + SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::with_description( + "Authorization", + "Session from sqb login; never combine with Cookie. Writes also require x-sqb-csrf.", + ))), + ); document .components .get_or_insert_with(Default::default) diff --git a/crates/sqlbuild-orchestration/src/api/main/parse_origin.rs b/crates/sqlbuild-orchestration/src/api/main/parse_origin.rs new file mode 100644 index 0000000000..74cc32b878 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/parse_origin.rs @@ -0,0 +1,28 @@ +//! Browser authentication and CLI handoffs share one strict HTTPS origin contract. + +use crate::api::constants::{HTTPS_SCHEME, ROOT_PATH}; +use crate::store::errors::ControlError; + +pub fn parse_origin(value: &str, mechanism: &str) -> Result { + let error = || { + ControlError::Invalid(format!( + "{mechanism} authentication requires an HTTPS origin without credentials, path, query or fragment" + )) + }; + let url = reqwest::Url::parse(value).map_err(|_| { + ControlError::Invalid(format!( + "{mechanism} authentication requires an HTTPS origin" + )) + })?; + if url.scheme() != HTTPS_SCHEME + || url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != ROOT_PATH + || url.query().is_some() + || url.fragment().is_some() + { + return Err(error()); + } + Ok(url.origin().ascii_serialization()) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index f51134549f..f48cfde010 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -2,9 +2,9 @@ use serde::{Deserialize, Serialize}; -use crate::api::_helpers::authentication::oidc; use crate::api::_helpers::authentication::password_sessions; use crate::api::_helpers::authentication::trusted_proxy; +use crate::api::_helpers::authentication::{cli_login, oidc}; use crate::api::_helpers::resources::{ administrators, audit, changes, location_health, location_work, person_location_work, person_operations, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, @@ -30,6 +30,9 @@ use crate::api::_helpers::resources::{ password_sessions::current, password_sessions::logout, password_sessions::change_password, + cli_login::start, + cli_login::approve, + cli_login::claim, trusted_proxy::identity, oidc::login, oidc::callback, @@ -246,6 +249,67 @@ pub struct PasswordSessionResponse { pub csrf_token: String, } +#[derive(Debug, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct CliLoginStartRequest { + pub(crate) client_hash: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum BrowserLoginMethod { + Password, + Oidc, + Proxy, +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct CliLoginApprovalRequest { + pub(crate) browser_code: String, + pub(crate) approve: bool, +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct CliLoginClaimRequest { + pub(crate) flow_id: uuid::Uuid, + pub(crate) client_secret: String, +} + +#[derive(Serialize, utoipa::ToSchema)] +pub(crate) struct CliLoginSessionResponse { + pub(crate) identity: crate::credentials::models::SessionIdentity, + pub(crate) bearer: String, + pub(crate) csrf_token: String, +} + +impl std::fmt::Debug for CliLoginApprovalRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CliLoginApprovalRequest") + .field("approve", &self.approve) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for CliLoginClaimRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CliLoginClaimRequest") + .field("flow_id", &self.flow_id) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for CliLoginSessionResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CliLoginSessionResponse") + .field("identity", &self.identity) + .finish_non_exhaustive() + } +} + impl std::fmt::Debug for PasswordSessionResponse { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs index 4ec849bd1f..05de59efd9 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::authorization::_helpers::administrators::{ deny_change, grant_installation_admin, remaining_administrator, }; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AdministratorAuthentication, AdministratorChange, AuditContext}; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs index 2c8140ca9a..747740c981 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::authorization::_helpers::administrator_changes::apply_change; use crate::authorization::main::deny_authorization::deny; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{ AdministratorAuthentication, AdministratorChange, AuditContext, BuiltinRole, RoleAssignmentRequest, diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs index a1307e181e..0556c412b8 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs @@ -4,7 +4,7 @@ use serde_json::json; use uuid::Uuid; use crate::authorization::main::deny_authorization::deny; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AdministratorAuthentication, AuditContext}; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs index e01232df14..2981ec2ec2 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs @@ -3,7 +3,7 @@ use serde_json::json; use crate::authorization::main::deny_authorization::deny; -use crate::credentials::main::authenticate_session_in::authenticate_session_in; +use crate::credentials::main::sessions::authenticate_session_in::authenticate_session_in; use crate::credentials::models::{ Action, AuditContext, BuiltinRole, DefaultRole, PermissionScope, SessionAccess, }; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs b/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs index 2e12d36608..b478ed53b4 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs @@ -4,8 +4,8 @@ use serde_json::json; use uuid::Uuid; use crate::authorization::_helpers::administrators::grant_installation_admin; -use crate::credentials::main::append_audit::append_audit; -use crate::credentials::main::insert_principal::insert_principal; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::identities::insert_principal::insert_principal; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs b/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs index 403164f51f..f1a76a2ca9 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs @@ -3,7 +3,7 @@ use serde_json::{Value, json}; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::commit::commit; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs index 50b5f64436..f61c435897 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/recover_administrator.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::authorization::_helpers::administrators::grant_installation_admin; use crate::authorization::main::deny_authorization::deny; -use crate::credentials::main::append_audit::append_audit; -use crate::credentials::main::revoke_session::revoke_session; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::sessions::revoke_session::revoke_session; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs new file mode 100644 index 0000000000..4c73c3fa38 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs @@ -0,0 +1,29 @@ +//! All person-session consumers share live person, expiry, revocation and password-version checks. + +use uuid::Uuid; + +use crate::credentials::models::StoredSession; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn active_session( + transaction: ControlTransaction, + id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.session_id,s.principal_id,s.expires_at,s.secret_hash,s.csrf_hash,s.credential_id FROM (\ + SELECT s.session_id,s.principal_id,s.expires_at,s.secret_hash,s.csrf_hash,s.revoked_at,s.credential_id \ + FROM sqb_password_sessions s JOIN sqb_password_credentials c \ + ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ + UNION ALL SELECT session_id,principal_id,expires_at,secret_hash,csrf_hash,revoked_at,NULL AS credential_id FROM sqb_external_sessions\ + ) s JOIN sqb_principals p ON p.principal_id=s.principal_id \ + WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ + AND p.kind='person' AND p.disabled_at IS NULL", + vec![Parameter::Id(id), Parameter::Time(now)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs index 1de5b30f42..d23ce57801 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use crate::credentials::_helpers::secrets::new_bearer; use crate::credentials::constants::PASSWORD_SESSION_SECONDS; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, IssuedSession, SessionIdentity}; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs index 672c0c5f70..06657a0414 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index d527e17e44..723c58f4d1 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod active_session; pub(crate) mod browser_sessions; pub(crate) mod locations; pub(crate) mod password_sessions; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs index d3163cad01..7be86f8194 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; use crate::credentials::_helpers::secrets::new_bearer; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/append_audit.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs rename to crates/sqlbuild-orchestration/src/credentials/main/audit/append_audit.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs new file mode 100644 index 0000000000..ef2b9bf9b0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod append_audit; +pub mod record_authentication_denial; +pub mod reserve_authentication_attempt; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/record_authentication_denial.rs similarity index 93% rename from crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs rename to crates/sqlbuild-orchestration/src/credentials/main/audit/record_authentication_denial.rs index c93861e4af..c79dbfa8ca 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/record_authentication_denial.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/record_authentication_denial.rs @@ -3,7 +3,7 @@ use serde_json::{Value, json}; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/reserve_authentication_attempt.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/reserve_authentication_attempt.rs similarity index 98% rename from crates/sqlbuild-orchestration/src/credentials/main/reserve_authentication_attempt.rs rename to crates/sqlbuild-orchestration/src/credentials/main/audit/reserve_authentication_attempt.rs index 434b0a4eb3..284d480b15 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/reserve_authentication_attempt.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/reserve_authentication_attempt.rs @@ -5,7 +5,7 @@ use serde_json::json; use uuid::Uuid; use crate::credentials::constants::{LOGIN_PEER_LIMIT, LOGIN_USER_LIMIT, LOGIN_WINDOW_SECONDS}; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, AuthenticationAttempt}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs deleted file mode 100644 index 824bf7b382..0000000000 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session_in.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Live person eligibility, password versions and revocation apply to every browser request. - -use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; -use crate::credentials::models::{SessionAccess, SessionIdentity, StoredSession}; -use crate::store::errors::ControlError; -use crate::store::main::{decode_row::decode_row, query::query}; -use crate::store::models::{ControlTransaction, Parameter}; - -pub(crate) async fn authenticate_session_in( - transaction: ControlTransaction, - bearer: &str, - access: SessionAccess<'_>, -) -> Result<(ControlTransaction, Option), ControlError> { - let Some(id) = bearer_id(bearer) else { - return Ok((transaction, None)); - }; - let now = transaction.now; - let (transaction, rows) = query(transaction, - "SELECT s.session_id, s.principal_id, s.expires_at, s.secret_hash, s.csrf_hash FROM (\ - SELECT s.session_id,s.principal_id,s.expires_at,s.secret_hash,s.csrf_hash,s.revoked_at \ - FROM sqb_password_sessions s JOIN sqb_password_credentials c \ - ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ - UNION ALL SELECT session_id,principal_id,expires_at,secret_hash,csrf_hash,revoked_at FROM sqb_external_sessions\ - ) s JOIN sqb_principals p ON p.principal_id=s.principal_id \ - WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ - AND p.kind='person' AND p.disabled_at IS NULL", - vec![Parameter::Id(id), Parameter::Time(now)]).await?; - let Some(row) = rows.into_iter().next() else { - return Ok((transaction, None)); - }; - let session: StoredSession = decode_row(row)?; - let csrf_matches = match access { - SessionAccess::Read => true, - SessionAccess::Write { csrf } => secret_matches(csrf, &session.csrf_hash), - }; - Ok(( - transaction, - (secret_matches(bearer, &session.secret_hash) && csrf_matches).then_some(session.identity), - )) -} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs new file mode 100644 index 0000000000..807129b2d6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs @@ -0,0 +1,71 @@ +//! Approval requires a live browser session and its CSRF proof inside the write fence. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::bearer_id; +use crate::credentials::main::{ + audit::append_audit::append_audit, sessions::authenticate_session_in::authenticate_session_in, +}; +use crate::credentials::models::{AuditContext, SessionAccess}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn approve_login( + store: &ControlStore, + context: &AuditContext, + (bearer, csrf): (&str, &str), + (browser_code, approve): (&str, bool), +) -> Result { + let Some(id) = bearer_id(browser_code) else { + return Ok(false); + }; + let (transaction, identity) = authenticate_session_in( + begin(store, TransactionMode::Write).await?, + bearer, + SessionAccess::Write { csrf }, + ) + .await?; + let Some(identity) = identity else { + return Ok(false); + }; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT flow_id FROM sqb_cli_login_flows WHERE flow_id=$1 AND browser_hash=$2 AND expires_at>$3 AND session_id IS NULL", + vec![Parameter::Id(id), Parameter::Text(sha256_hex(browser_code.as_bytes())), Parameter::Time(now)]).await?; + if rows.is_empty() { + return Ok(false); + } + let (transaction, _) = if approve { + query( + transaction, + "UPDATE sqb_cli_login_flows SET session_id=$2 WHERE flow_id=$1", + vec![Parameter::Id(id), Parameter::Id(identity.session_id)], + ) + .await? + } else { + query( + transaction, + "DELETE FROM sqb_cli_login_flows WHERE flow_id=$1", + vec![Parameter::Id(id)], + ) + .await? + }; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"decision"), "cli.login.decided"), + ( + json!({"flow_id":id,"principal_id":identity.principal_id}), + json!(null), + json!({"approved":approve}), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs new file mode 100644 index 0000000000..152448841a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs @@ -0,0 +1,69 @@ +//! Exchange consumes approval and issues a normal bounded person session atomically. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::{ + active_session::active_session, browser_sessions::issue_session, secrets::secret_matches, +}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, ClaimedLogin, StoredLogin}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn claim_login( + store: &ControlStore, + context: &AuditContext, + (id, client_secret): (Uuid, &str), +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT client_hash,session_id FROM sqb_cli_login_flows WHERE flow_id=$1 AND expires_at>$2", + vec![Parameter::Id(id), Parameter::Time(now)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + return Ok(ClaimedLogin::Unavailable); + }; + let flow: StoredLogin = decode_row(row)?; + if !secret_matches(client_secret, &flow.client_hash) { + return Ok(ClaimedLogin::Unavailable); + } + let Some(parent) = flow.session_id else { + return Ok(ClaimedLogin::Pending); + }; + let (transaction, parent) = active_session(transaction, parent).await?; + let Some(parent) = parent else { + return Ok(ClaimedLogin::Unavailable); + }; + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_cli_login_flows WHERE flow_id=$1", + vec![Parameter::Id(id)], + ) + .await?; + let (transaction, session) = issue_session( + transaction, + context, + (parent.identity.principal_id, parent.credential_id), + ) + .await?; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"exchanged"), "cli.login.exchanged"), + ( + json!({"flow_id":id,"principal_id":session.identity.principal_id}), + json!(null), + json!({"session_id":session.identity.session_id}), + ), + ) + .await?, + ) + .await?; + Ok(ClaimedLogin::Authenticated(session)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs new file mode 100644 index 0000000000..e46edbe52f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod approve_login; +pub(crate) mod claim_login; +pub(crate) mod start_login; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs new file mode 100644 index 0000000000..77cb358898 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs @@ -0,0 +1,57 @@ +//! Only a digest of the CLI-held exchange secret is sent when starting a login. + +use chrono::{Duration, SubsecRound}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::constants::{OIDC_FLOW_SECONDS, TOKEN_SECRET_HEX_LENGTH}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, StartedLogin}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn start_login( + store: &ControlStore, + context: &AuditContext, + client_hash: &str, +) -> Result { + if client_hash.len() != TOKEN_SECRET_HEX_LENGTH + || !client_hash.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(ControlError::Invalid("invalid CLI exchange digest".into())); + } + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let id = Uuid::now_v7(); + let browser_code = new_bearer(id)?; + let expires_at = (now + Duration::seconds(OIDC_FLOW_SECONDS)).trunc_subsecs(6); + let (transaction, _) = query(transaction, + "DELETE FROM sqb_cli_login_flows WHERE flow_id IN \ + (SELECT flow_id FROM sqb_cli_login_flows WHERE expires_at<=$1 ORDER BY expires_at LIMIT 1000)", + vec![Parameter::Time(now)]).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_cli_login_flows (flow_id,client_hash,browser_hash,expires_at) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(id), Parameter::Text(client_hash.into()), + Parameter::Text(sha256_hex(browser_code.as_bytes())), Parameter::Time(expires_at)]).await?; + commit( + append_audit( + transaction, + context, + (id, "cli.login.started"), + ( + json!({"flow_id":id}), + json!(null), + json!({"expires_at":expires_at}), + ), + ) + .await?, + ) + .await?; + Ok(StartedLogin { + browser_code, + expires_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/identities/create_principal.rs similarity index 89% rename from crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs rename to crates/sqlbuild-orchestration/src/credentials/main/identities/create_principal.rs index 153fd94d6e..17689de4d7 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/identities/create_principal.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::credentials::main::insert_principal::insert_principal; +use crate::credentials::main::identities::insert_principal::insert_principal; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/identities/insert_principal.rs similarity index 95% rename from crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs rename to crates/sqlbuild-orchestration/src/credentials/main/identities/insert_principal.rs index d8d2c95e5b..919aebda53 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/insert_principal.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/identities/insert_principal.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs new file mode 100644 index 0000000000..54451634e7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs @@ -0,0 +1,2 @@ +pub mod create_principal; +pub(crate) mod insert_principal; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/create_location.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/create_location.rs rename to crates/sqlbuild-orchestration/src/credentials/main/locations/create_location.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/development_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/development_location.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/development_location.rs rename to crates/sqlbuild-orchestration/src/credentials/main/locations/development_location.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs new file mode 100644 index 0000000000..23453c831d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs @@ -0,0 +1,2 @@ +pub mod create_location; +pub mod development_location; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs index 7416722370..06dec347c3 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -1,20 +1,6 @@ -pub(crate) mod append_audit; -pub(crate) mod authenticate_bound_person; -pub mod authenticate_session; -pub(crate) mod authenticate_session_in; -pub mod authenticate_token; -pub(crate) mod claim_oidc_flow; -pub mod create_location; -pub(crate) mod create_oidc_flow; -pub mod create_principal; -pub mod development_location; -pub(crate) mod insert_principal; -pub mod issue_token; -pub mod login_password; -pub mod record_authentication_denial; -pub mod reserve_authentication_attempt; -pub mod revoke_session; -pub mod revoke_token; -pub mod rotate_token; -pub mod set_password; -pub mod verify_password; +pub mod audit; +pub(crate) mod cli_login; +pub mod identities; +pub mod locations; +pub mod sessions; +pub mod tokens; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_bound_person.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_bound_person.rs index 041c01a3ba..266336964e 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_bound_person.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_bound_person.rs @@ -4,7 +4,7 @@ use serde_json::json; use uuid::Uuid; use crate::credentials::_helpers::browser_sessions::issue_session; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, ExternalIdentityMechanism, IssuedSession}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session.rs similarity index 87% rename from crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session.rs index ee30024156..273fd28904 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_session.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session.rs @@ -1,7 +1,7 @@ //! Read authentication and mutations share the same live session/version checks. use crate::credentials::_helpers::secrets::bearer_id; -use crate::credentials::main::authenticate_session_in::authenticate_session_in as authenticate; +use crate::credentials::main::sessions::authenticate_session_in::authenticate_session_in as authenticate; use crate::credentials::models::{SessionAccess, SessionIdentity}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session_in.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session_in.rs new file mode 100644 index 0000000000..7708ee8deb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session_in.rs @@ -0,0 +1,29 @@ +//! Live person eligibility, password versions and revocation apply to every browser request. + +use crate::credentials::_helpers::active_session::active_session; +use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; +use crate::credentials::models::{SessionAccess, SessionIdentity}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn authenticate_session_in( + transaction: ControlTransaction, + bearer: &str, + access: SessionAccess<'_>, +) -> Result<(ControlTransaction, Option), ControlError> { + let Some(id) = bearer_id(bearer) else { + return Ok((transaction, None)); + }; + let (transaction, session) = active_session(transaction, id).await?; + let Some(session) = session else { + return Ok((transaction, None)); + }; + let csrf_matches = match access { + SessionAccess::Read => true, + SessionAccess::Write { csrf } => secret_matches(csrf, &session.csrf_hash), + }; + Ok(( + transaction, + (secret_matches(bearer, &session.secret_hash) && csrf_matches).then_some(session.identity), + )) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/claim_oidc_flow.rs similarity index 97% rename from crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/claim_oidc_flow.rs index b02f92452c..4420e480e8 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/claim_oidc_flow.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/claim_oidc_flow.rs @@ -5,7 +5,7 @@ use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::credentials::_helpers::secrets::secret_matches; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, StoredOidcFlow}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/create_oidc_flow.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/create_oidc_flow.rs index 83eb7afcf8..c001b9ec53 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/create_oidc_flow.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/create_oidc_flow.rs @@ -1,7 +1,7 @@ //! Create one-use browser-bound PKCE attempts with durable audit evidence. use crate::credentials::constants::OIDC_FLOW_SECONDS; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, OidcFlowStart}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/login_password.rs similarity index 89% rename from crates/sqlbuild-orchestration/src/credentials/main/login_password.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/login_password.rs index 2f86ab6cab..d1a41f04de 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/login_password.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/login_password.rs @@ -4,9 +4,9 @@ use uuid::Uuid; use crate::credentials::_helpers::password_sessions::issue_session; use crate::credentials::constants::{LOGIN_USERNAME_BYTES, MAX_PASSWORD_BYTES}; -use crate::credentials::main::record_authentication_denial::record_authentication_denial; -use crate::credentials::main::reserve_authentication_attempt::reserve_authentication_attempt; -use crate::credentials::main::verify_password::verify_password; +use crate::credentials::main::audit::record_authentication_denial::record_authentication_denial; +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::sessions::verify_password::verify_password; use crate::credentials::models::{AuditContext, AuthenticationAttempt, PasswordLogin}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs new file mode 100644 index 0000000000..6f7d33ccab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod authenticate_bound_person; +pub mod authenticate_session; +pub(crate) mod authenticate_session_in; +pub(crate) mod claim_oidc_flow; +pub(crate) mod create_oidc_flow; +pub mod login_password; +pub(crate) mod revoke_session; +pub mod set_password; +pub mod verify_password; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/revoke_session.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/revoke_session.rs index bbac342acf..1f09236983 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/revoke_session.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/revoke_session.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::store::errors::ControlError; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/set_password.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/credentials/main/set_password.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/set_password.rs index 9e68d961f7..ea636e892a 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/set_password.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/set_password.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::credentials::_helpers::passwords::hash_password; use crate::credentials::constants::{MAX_PASSWORD_BYTES, MIN_PASSWORD_CHARACTERS}; -use crate::credentials::main::append_audit::append_audit; -use crate::credentials::main::revoke_session::revoke_session; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::sessions::revoke_session::revoke_session; use crate::credentials::models::{AuditContext, PasswordAuthority}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/verify_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/verify_password.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/verify_password.rs rename to crates/sqlbuild-orchestration/src/credentials/main/sessions/verify_password.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/authenticate_token.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs rename to crates/sqlbuild-orchestration/src/credentials/main/tokens/authenticate_token.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/issue_token.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs rename to crates/sqlbuild-orchestration/src/credentials/main/tokens/issue_token.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs new file mode 100644 index 0000000000..607195c76d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs @@ -0,0 +1,4 @@ +pub mod authenticate_token; +pub mod issue_token; +pub mod revoke_token; +pub mod rotate_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/revoke_token.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs rename to crates/sqlbuild-orchestration/src/credentials/main/tokens/revoke_token.rs diff --git a/crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/rotate_token.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs rename to crates/sqlbuild-orchestration/src/credentials/main/tokens/rotate_token.rs index cc93a374ff..a458a9d5ce 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/rotate_token.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/rotate_token.rs @@ -4,7 +4,7 @@ use serde_json::json; use uuid::Uuid; use crate::credentials::_helpers::tokens::{issue, revoke}; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{AuditContext, IssuedToken, TokenIdentity}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index 4a3fcc411f..1458654cbf 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -192,6 +192,34 @@ pub struct IssuedSession { pub csrf: String, } +#[derive(Serialize, utoipa::ToSchema)] +pub(crate) struct StartedLogin { + pub(crate) browser_code: String, + pub(crate) expires_at: DateTime, +} + +#[derive(Debug)] +pub(crate) enum ClaimedLogin { + Pending, + Unavailable, + Authenticated(IssuedSession), +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredLogin { + pub(crate) client_hash: String, + pub(crate) session_id: Option, +} + +impl std::fmt::Debug for StartedLogin { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("StartedLogin") + .field("expires_at", &self.expires_at) + .finish_non_exhaustive() + } +} + impl std::fmt::Debug for IssuedSession { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter @@ -241,6 +269,7 @@ pub(crate) struct StoredOidcFlow { pub(crate) struct StoredSession { #[serde(flatten)] pub(crate) identity: SessionIdentity, + pub(crate) credential_id: Option, pub(crate) secret_hash: String, pub(crate) csrf_hash: String, } diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs index 82ea7acec7..1fc258e2d0 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs @@ -2,7 +2,7 @@ use serde_json::json; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::declarations::_helpers::operator_state::apply_operator_change; use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index 7c3b8698af..7c2ac04f72 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -4,7 +4,7 @@ use chrono::Duration; use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::platform::_helpers::connection_tests::authorize_connection_test; use crate::platform::main::read_location_work::read_location_work; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs index bc2e969b69..05010ce95b 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs @@ -2,7 +2,7 @@ use serde_json::json; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::runs::_helpers::lifecycle::changes::{ check_launch_claim, validate_authority, validate_change, validate_retry, }; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index 33875643e1..1477cd8a15 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -4,7 +4,7 @@ use serde_json::json; use uuid::Uuid; use crate::authorization::main::authorize_session::authorize_session; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs index 2c98f2f9b2..d9e48554c8 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs @@ -3,7 +3,7 @@ use serde_json::json; use uuid::Uuid; -use crate::credentials::main::append_audit::append_audit; +use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext}; use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 66bc5198e7..d6a956106d 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -41,6 +41,7 @@ pub(crate) const FINGERPRINT_LOOKUP: &str = include_str!("../fingerprint_lookup. pub(crate) const WAREHOUSE_STATISTICS: &str = include_str!("../warehouse_statistics.sql"); pub(crate) const CHECK_OBSERVATIONS: &str = include_str!("../check_observations.sql"); pub(crate) const TRIGGER_PROGRESS: &str = include_str!("../trigger_progress.sql"); +pub(crate) const CLI_LOGIN_FLOWS: &str = include_str!("../cli_login_flows.sql"); pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("location_instances", LOCATION_INSTANCES), @@ -74,6 +75,7 @@ pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("warehouse_statistics", WAREHOUSE_STATISTICS), ("check_observations", CHECK_OBSERVATIONS), ("trigger_progress", TRIGGER_PROGRESS), + ("cli_login_flows", CLI_LOGIN_FLOWS), ]; pub(crate) fn migration_digests(store: &ControlStore) -> Vec { diff --git a/crates/sqlbuild-orchestration/src/store/cli_login_flows.sql b/crates/sqlbuild-orchestration/src/store/cli_login_flows.sql new file mode 100644 index 0000000000..bffde896b1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/cli_login_flows.sql @@ -0,0 +1,8 @@ +CREATE TABLE sqb_cli_login_flows ( + flow_id {uuid} PRIMARY KEY, + client_hash TEXT NOT NULL, + browser_hash TEXT NOT NULL, + expires_at {time} NOT NULL, + session_id {uuid} +); +CREATE INDEX sqb_cli_login_expiry ON sqb_cli_login_flows(expires_at); diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index 529087b313..007b0a39b8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -8,7 +8,8 @@ use crate::authorization::main::{ assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, }; use crate::credentials::main::{ - create_principal::create_principal, login_password::login_password, set_password::set_password, + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, }; use crate::credentials::models::{ AdministratorAuthentication, AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs index f679fc3b92..6b74d5a9d7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_changes.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -12,7 +12,7 @@ use crate::api::tests::changes_helpers::{ }; use crate::api::tests::helpers::{read_json, start_api, stop_api}; use crate::api::tests::test_types::ChangesTestCase; -use crate::credentials::main::{issue_token::issue_token, revoke_token::revoke_token}; +use crate::credentials::main::tokens::{issue_token::issue_token, revoke_token::revoke_token}; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_cli_login.rs b/crates/sqlbuild-orchestration/tests/api/test_cli_login.rs new file mode 100644 index 0000000000..3705c7a587 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_cli_login.rs @@ -0,0 +1,396 @@ +//! Real cross-replica exchange, atomic audit rollback and live password/session revocation. + +use futures_util::future::join_all; +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::api::models::ApiAuthentication; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::CliLoginTestCase; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::revoke_session::revoke_session, sessions::set_password::set_password, +}; +use crate::credentials::models::{AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_browser_approved_cli_login_when_exchange_races_or_authority_changes_then_credentials_stay_single_use_atomic_and_live() + { + let test_cases = [ + CliLoginTestCase { + description: "SQLite single-use browser handoff", + backend: Backend::Sqlite, + expected_exchanges: 1, + }, + CliLoginTestCase { + description: "PostgreSQL single-use browser handoff", + backend: Backend::Postgres, + expected_exchanges: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/cli_login.json"); + let input: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("inputs"); + let password = input["password"].as_str().expect("password"); + let replacement = input["replacement"].as_str().expect("replacement"); + let origin = input["origin"].as_str().expect("origin"); + let host = origin.strip_prefix("https://").expect("HTTPS"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("peer")), + }; + let person = + create_principal(store, &context, ("orders-reader", PrincipalKind::Person)) + .await + .expect("person"); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(browser) = + login_password(store, &context, ("orders-reader", password)) + .await + .expect("browser login") + else { + panic!("browser credentials"); + }; + let authentication = || ApiAuthentication { + password_origin: Some(origin.into()), + ..ApiAuthentication::default() + }; + let first = start_api_with_authentication(store, authentication()).await; + let second = start_api_with_authentication(&replica, authentication()).await; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert("host", host.parse().expect("host")); + headers.insert("origin", origin.parse().expect("origin")); + let client = Client::builder() + .default_headers(headers) + .build() + .expect("client"); + let secret = input["client_secret"] + .as_str() + .expect("synthetic exchange secret"); + let started = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("start"); + assert_eq!(started.status(), StatusCode::OK); + assert_eq!(started.headers()["cache-control"], "no-store"); + let flow: Value = started.json().await.expect("handoff"); + let code = flow["browser_code"].as_str().expect("browser code"); + let flow_id = code.split_once('.').expect("flow ID").0; + let exchange = json!({"flow_id":flow_id,"client_secret":secret}); + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", second.url)) + .json(&exchange) + .send() + .await + .expect("pending") + .status(), + StatusCode::ACCEPTED + ); + let approval = json!({"browser_code":code,"approve":true}); + let cookie = format!("__Host-sqb-session={}", browser.bearer); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .json(&approval) + .send() + .await + .expect("missing CSRF") + .status(), + StatusCode::UNAUTHORIZED + ); + let cancelled: Value = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("cancel start") + .json() + .await + .expect("cancel flow"); + let cancelled_code = cancelled["browser_code"].as_str().expect("cancel code"); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&json!({"browser_code":cancelled_code,"approve":false})) + .send() + .await + .expect("cancel") + .status(), + StatusCode::NO_CONTENT + ); + assert_eq!(client.post(format!("{}/api/auth/cli/claim", first.url)) + .json(&json!({"flow_id":cancelled_code.split_once('.').expect("cancel ID").0,"client_secret":secret})) + .send().await.expect("cancel claim").status(), StatusCode::UNAUTHORIZED); + let expired: Value = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("expiry start") + .json() + .await + .expect("expiry flow"); + let expired_code = expired["browser_code"].as_str().expect("expiry code"); + let expired_id = + Uuid::parse_str(expired_code.split_once('.').expect("expiry ID").0).expect("ID"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("expiry fence"); + let expired_at = transaction.now - chrono::Duration::seconds(1); + let (transaction, _) = query( + transaction, + "UPDATE sqb_cli_login_flows SET expires_at=$2 WHERE flow_id=$1", + vec![Parameter::Id(expired_id), Parameter::Time(expired_at)], + ) + .await + .expect("elapsed expiry"); + commit(transaction).await.expect("commit expiry"); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&json!({"browser_code":expired_code,"approve":true})) + .send() + .await + .expect("expired approval") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", first.url)) + .json(&json!({"flow_id":expired_id,"client_secret":secret})) + .send() + .await + .expect("expired claim") + .status(), + StatusCode::UNAUTHORIZED + ); + let approved = client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&approval) + .send() + .await + .expect("approve"); + assert_eq!(approved.status(), StatusCode::NO_CONTENT); + assert!(!approved.headers().contains_key("set-cookie")); + assert!(approved.bytes().await.expect("approval body").is_empty()); + let wrong = json!({"flow_id":flow_id,"client_secret":"not-the-cli-secret"}); + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", first.url)) + .json(&wrong) + .send() + .await + .expect("wrong exchange") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "CLI exchange audit rollback", + table: "sqb_security_audit", + operation: "INSERT", + condition: "WHEN NEW.action='cli.login.exchanged'", + sqlite_body: "SELECT RAISE(ABORT, 'CLI exchange audit failed')", + postgres_body: "RAISE EXCEPTION 'CLI exchange audit failed'", + }; + install_fault(store, &fault).await; + let before = logical_state(store).await; + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", first.url)) + .json(&exchange) + .send() + .await + .expect("failed exchange") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + logical_state(store).await, + before, + "failed exchange preserves every logical row" + ); + remove_fault(store, &fault).await; + let attempts = join_all((0..4).map(|index| { + let client = &client; + let exchange = &exchange; + let url = [&first.url, &second.url][index % 2]; + async move { + client + .post(format!("{url}/api/auth/cli/claim")) + .json(exchange) + .send() + .await + .expect("concurrent exchange") + } + })) + .await; + assert_eq!( + attempts + .iter() + .filter(|response| response.status() == StatusCode::OK) + .count(), + test_case.expected_exchanges, + "{}", + test_case.description + ); + assert_eq!( + attempts + .iter() + .filter(|response| response.status() == StatusCode::UNAUTHORIZED) + .count(), + 3 + ); + let issued: Value = attempts + .into_iter() + .find(|response| response.status() == StatusCode::OK) + .expect("one credential") + .json() + .await + .expect("CLI credential"); + assert_eq!(issued["identity"]["principal_id"], person.to_string()); + let authorization = format!("Session {}", issued["bearer"].as_str().expect("bearer")); + for url in [&first.url, &second.url] { + assert_eq!( + client + .get(format!("{url}/api/auth/session")) + .header("authorization", &authorization) + .send() + .await + .expect("CLI reload transport") + .status(), + StatusCode::OK + ); + assert_eq!( + client + .get(format!("{url}/api/control/audit")) + .header("authorization", &authorization) + .send() + .await + .expect("no implicit grant") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{url}/api/auth/logout")) + .header("authorization", &authorization) + .send() + .await + .expect("CLI CSRF required") + .status(), + StatusCode::UNAUTHORIZED + ); + } + assert_eq!( + client + .get(format!("{}/api/auth/session", first.url)) + .header("authorization", &authorization) + .header("cookie", &cookie) + .send() + .await + .expect("ambiguous transport") + .status(), + StatusCode::UNAUTHORIZED + ); + let started: Value = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("next start") + .json() + .await + .expect("next flow"); + let next_code = started["browser_code"].as_str().expect("next code"); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", first.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&json!({"browser_code":next_code,"approve":true})) + .send() + .await + .expect("next approve") + .status(), + StatusCode::NO_CONTENT + ); + let (transaction, _) = revoke_session( + begin(store, TransactionMode::Write).await.expect("write"), + &context, + (person, Some(browser.identity.session_id)), + ) + .await + .expect("revoke browser"); + commit(transaction).await.expect("commit revoke"); + assert_eq!(client.post(format!("{}/api/auth/cli/claim", second.url)) + .json(&json!({"flow_id":next_code.split_once('.').expect("next ID").0,"client_secret":secret})) + .send().await.expect("revoked authorizer").status(), StatusCode::UNAUTHORIZED); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement, + ) + .await + .expect("rotate password"); + for url in [&first.url, &second.url] { + assert_eq!( + client + .get(format!("{url}/api/auth/session")) + .header("authorization", &authorization) + .send() + .await + .expect("live password revocation") + .status(), + StatusCode::UNAUTHORIZED + ); + } + stop_api(first).await; + stop_api(second).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 53c0771e14..7631ca41d1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use std::time::Duration; use crate::api::errors::ClientError; -use crate::credentials::main::issue_token::issue_token; -use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::main::tokens::revoke_token::revoke_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::runs::main::change_run::change_run; use crate::runs::main::claim_next_run::claim_next_run; @@ -114,6 +114,19 @@ fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retrie openapi["components"]["securitySchemes"]["bearer"]["scheme"], "bearer" ); + assert_eq!( + openapi["components"]["securitySchemes"]["cliSession"]["name"], + "Authorization" + ); + assert_eq!( + openapi["paths"]["/api/control/runs"]["post"]["security"], + json!([{"session": []}, {"cliSession": []}]) + ); + assert_eq!( + openapi["paths"]["/api/auth/cli/approve"]["post"]["security"], + json!([{"session": []}]), + "browser authorization cannot use a CLI session header" + ); assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 62dde71a90..d8de3cff39 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -30,6 +30,8 @@ mod statistics_retention; mod test_catalog_reads; #[path = "test_changes.rs"] mod test_changes; +#[path = "test_cli_login.rs"] +mod test_cli_login; #[path = "test_contract.rs"] mod test_contract; #[path = "test_development.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_development.rs b/crates/sqlbuild-orchestration/tests/api/test_development.rs index 3540299f62..d7dd3a7f40 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_development.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_development.rs @@ -3,7 +3,7 @@ use uuid::Uuid; use crate::api::models::{ApiAuthentication, DevelopmentScope}; -use crate::credentials::main::development_location::development_location; +use crate::credentials::main::locations::development_location::development_location; use crate::credentials::models::AuditContext; use crate::declarations::main::{ change_trigger::change_trigger, list_triggers::list_triggers, diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs index 9960f27274..deeedc2e7a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -11,8 +11,8 @@ use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::ExternalSessionTestCase; use crate::authorization::main::bootstrap_user::bootstrap_user; use crate::credentials::main::{ - create_location::create_location, create_principal::create_principal, - set_password::set_password, verify_password::verify_password, + identities::create_principal::create_principal, locations::create_location::create_location, + sessions::set_password::set_password, sessions::verify_password::verify_password, }; use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; use crate::store::main::{ diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs index 002eed46e0..db5ebfe4fa 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -1,4 +1,4 @@ -use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::platform::main::report_remote_location_health::report_remote_location_health; use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 2701c9171b..60847329be 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -1,5 +1,5 @@ use crate::api::errors::ClientError; -use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::platform::main::connect_location_work::connect_location_work; use crate::platform::main::location_work::location_work; diff --git a/crates/sqlbuild-orchestration/tests/api/test_oidc.rs b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs index 7b484e20fa..e06418240c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_oidc.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs @@ -11,7 +11,7 @@ use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::oidc_helpers::{callback, login, start_provider, stop_provider}; use crate::api::tests::oidc_test_types::{BrowserFlow, ProviderFault}; use crate::api::tests::test_types::{OidcApiTestCase, OidcThrottleTestCase}; -use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/api/test_passwords.rs b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs index 63fc9fd73d..ba84bcc7be 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_passwords.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs @@ -8,7 +8,9 @@ use uuid::Uuid; use crate::api::models::{ApiAuthentication, AuthenticationDenial, PasswordSessionResponse}; use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::PasswordApiTestCase; -use crate::credentials::main::{create_principal::create_principal, set_password::set_password}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::set_password::set_password, +}; use crate::credentials::models::{AuditContext, DefaultRole, PasswordAuthority, PrincipalKind}; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index 94422c144e..07a08d5761 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -7,7 +7,8 @@ use crate::authorization::main::{ assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, }; use crate::credentials::main::{ - create_principal::create_principal, login_password::login_password, set_password::set_password, + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, }; use crate::credentials::models::{ AdministratorAuthentication, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index c87ae8d35a..376616db8e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -13,7 +13,8 @@ use crate::authorization::main::{ assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, }; use crate::credentials::main::{ - create_principal::create_principal, login_password::login_password, set_password::set_password, + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, }; use crate::credentials::models::{ AdministratorAuthentication, AuditContext, PasswordAuthority, PasswordLogin, PrincipalKind, diff --git a/crates/sqlbuild-orchestration/tests/api/test_proxy.rs b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs index 5d3ca7a363..a70f507917 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_proxy.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs @@ -9,7 +9,7 @@ use uuid::Uuid; use crate::api::models::{ApiAuthentication, AuthenticationDenial, TrustedProxyAuthentication}; use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::ProxyApiTestCase; -use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::{AuditContext, PrincipalKind}; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index c1c6eaa049..352df748dd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -3,9 +3,9 @@ use std::collections::BTreeMap; use std::time::Duration; -use crate::credentials::main::create_principal::create_principal; -use crate::credentials::main::issue_token::issue_token; -use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::main::tokens::revoke_token::revoke_token; use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; use crate::declarations::main::change_trigger::change_trigger; use crate::declarations::main::list_triggers::list_triggers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index ee1b9619be..09e69bb3d9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -105,3 +105,9 @@ pub(crate) struct OidcThrottleTestCase { pub(crate) backend: Backend, pub(crate) expected_allowed: usize, } + +pub(crate) struct CliLoginTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_exchanges: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index beb5f9c29c..8ab85c3b95 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -8,7 +8,7 @@ use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; use crate::api::tests::helpers::{start_api, stop_api}; use crate::api::tests::test_types::WarehouseMetadataTestCase; -use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; use crate::platform::main::connect_location_work::connect_location_work; diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index e9d9e1aa80..0d21031a7c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::runs::main::change_run::change_run; use crate::runs::main::claim_next_run::claim_next_run; diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs index 9a48e51758..43e31882ef 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -10,7 +10,7 @@ pub(crate) async fn create_location( name: &str, actor: &str, ) -> Result { - crate::credentials::main::create_location::create_location( + crate::credentials::main::locations::create_location::create_location( store, name, &AuditContext { diff --git a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs index 12dc42386d..ee47e93b7e 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs @@ -4,8 +4,8 @@ use serde_json::{Value, json}; use uuid::Uuid; use crate::credentials::main::{ - create_principal::create_principal, set_password::set_password, - verify_password::verify_password, + identities::create_principal::create_principal, sessions::set_password::set_password, + sessions::verify_password::verify_password, }; use crate::credentials::models::{AuditContext, PasswordAuthority, PrincipalKind}; use crate::credentials::tests::helpers::authentication_revision; diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs index 54743a7b94..0c1102bc24 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs @@ -6,10 +6,11 @@ use serde_json::{Value, json}; use uuid::Uuid; use crate::credentials::main::{ - authenticate_session::authenticate_session, authenticate_token::authenticate_token, - create_principal::create_principal, login_password::login_password, - reserve_authentication_attempt::reserve_authentication_attempt, revoke_session::revoke_session, - set_password::set_password, verify_password::verify_password, + audit::reserve_authentication_attempt::reserve_authentication_attempt, + identities::create_principal::create_principal, + sessions::authenticate_session::authenticate_session, sessions::login_password::login_password, + sessions::revoke_session::revoke_session, sessions::set_password::set_password, + sessions::verify_password::verify_password, tokens::authenticate_token::authenticate_token, }; use crate::credentials::models::{ AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, PrincipalKind, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs index 4790b6db06..01e4db0301 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs @@ -8,8 +8,8 @@ use crate::authorization::main::{ bootstrap_user::bootstrap_user, change_administrator::change_administrator, }; use crate::credentials::main::{ - create_principal::create_principal, login_password::login_password, - revoke_session::revoke_session, set_password::set_password, + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::revoke_session::revoke_session, sessions::set_password::set_password, }; use crate::credentials::models::{ AdministratorAuthentication, AdministratorChange, AuditContext, PasswordAuthority, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 175c166145..65f5fb076b 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -19,8 +19,8 @@ use crate::authorization::main::{ read_security_audit::read_security_audit, }; use crate::credentials::main::{ - append_audit::append_audit, create_principal::create_principal, login_password::login_password, - set_password::set_password, + audit::append_audit::append_audit, identities::create_principal::create_principal, + sessions::login_password::login_password, sessions::set_password::set_password, }; use crate::credentials::models::{ AdministratorAuthentication, AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index bcb86ea674..ab8c52ba3a 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -1,11 +1,11 @@ use std::collections::BTreeMap; -use crate::credentials::main::authenticate_token::authenticate_token; -use crate::credentials::main::create_location::create_location; -use crate::credentials::main::create_principal::create_principal; -use crate::credentials::main::issue_token::issue_token; -use crate::credentials::main::revoke_token::revoke_token; -use crate::credentials::main::rotate_token::rotate_token; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::main::locations::create_location::create_location; +use crate::credentials::main::tokens::authenticate_token::authenticate_token; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::main::tokens::revoke_token::revoke_token; +use crate::credentials::main::tokens::rotate_token::rotate_token; use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; use crate::runs::main::claim_next_run::claim_next_run; use crate::scheduler::tests::contracts::helpers::acquire_lease; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs index bfea637a46..74bc357b1e 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs @@ -9,9 +9,10 @@ use crate::authorization::main::{ bootstrap_user::bootstrap_user, recover_administrator::recover_administrator, }; use crate::credentials::main::{ - authenticate_bound_person::authenticate_bound_person, - authenticate_session::authenticate_session, create_principal::create_principal, - login_password::login_password, set_password::set_password, + identities::create_principal::create_principal, + sessions::authenticate_bound_person::authenticate_bound_person, + sessions::authenticate_session::authenticate_session, sessions::login_password::login_password, + sessions::set_password::set_password, }; use crate::credentials::models::{ AuditContext, ExternalIdentityMechanism, PasswordAuthority, PasswordLogin, PrincipalKind, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index 32859f25fd..b1cf015f69 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -8,7 +8,8 @@ use crate::authorization::main::{ change_administrator::change_administrator, }; use crate::credentials::main::{ - create_principal::create_principal, login_password::login_password, set_password::set_password, + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, }; use crate::credentials::models::{ AdministratorAuthentication, AdministratorChange, AuditContext, BuiltinRole, DefaultRole, diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs index e3407c4419..570801654b 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -7,7 +7,7 @@ use serde_json::json; use uuid::Uuid; use crate::api::tests::helpers::{start_api, stop_api}; -use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index 55d56b7fd8..b46ed0da1e 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -92,6 +92,8 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DROP TABLE sqb_cli_login_flows", + "DELETE FROM sqb_schema_migrations WHERE name='cli_login_flows'", "DROP TABLE sqb_trigger_progress", "DELETE FROM sqb_schema_migrations WHERE name='trigger_progress'", "DROP INDEX sqb_observed_node_cursor", diff --git a/fensu.toml b/fensu.toml index eb37274291..ecc2f19c8d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1530,6 +1530,11 @@ rule = "FFR601" path = "src/sqlbuild/executor/run/_helpers/materializations/microbatch.py" reason = "Microbatch execution is one ordered failure-sensitive lifecycle; splitting reconciliation, DML, completion publication, and aggregate gates would hide the required phase ordering across module boundaries." +[[targets.python.rule_exceptions]] +rule = "FFT104" +path = "tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py" +reason = "The disposable HTTPS gateway and OIDC provider dispatch real HTTP protocol messages and authentication transports; assertions in the parametrized installed journey remain branch-free." + [targets.rust] analyzer = "rust" roots = ["crates"] @@ -1567,15 +1572,21 @@ reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot imp [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/credentials/models.rs" -symbols = ["IssuedToken", "IssuedSession"] -reason = "Issued credentials implement Debug manually to redact bearer and CSRF secrets; deriving Debug would expose them." +symbols = ["IssuedToken", "IssuedSession", "StartedLogin"] +reason = "Issued credentials and handoffs implement Debug manually to redact bearer, CSRF and approval secrets; deriving Debug would expose them." [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/api/models.rs" -symbols = ["PasswordLoginRequest", "PasswordChangeRequest", "PasswordSessionResponse", "ExternalIdentityResponse"] +symbols = ["PasswordLoginRequest", "PasswordChangeRequest", "PasswordSessionResponse", "ExternalIdentityResponse", "CliLoginApprovalRequest", "CliLoginClaimRequest", "CliLoginSessionResponse"] reason = "Browser transport models implement Debug manually to redact passwords and CSRF secrets; other models derive Debug." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-cli/src/person_login/models.rs" +symbols = ["PersonCredential", "LoginFlow"] +reason = "Person credential and handoff models implement Debug manually to redact bearer, CSRF and approval secrets." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/webhooks/models.rs" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 131fd6acea..7b27967fec 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -115,6 +115,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb playground](cli/playground.md) (`cli/playground`) - Create a self-contained SQLBuild project to explore locally. - [sqb skills](cli/skills.md) (`cli/skills`) - Install SQLBuild skill files for AI coding agents. - [sqb server](cli/server.md) (`cli/server`) - Run the control service or repair its derived views. +- [sqb login and logout](cli/login.md) (`cli/login`) - Authorize a bounded CLI session through your server's browser sign-in. - [sqb compile](cli/compile.md) (`cli/compile`) - Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. - [sqb contract](cli/contract.md) (`cli/contract`) - Compare or generate repository contracts from physical warehouse schemas. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md new file mode 100644 index 0000000000..eb42ef116e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md @@ -0,0 +1,65 @@ + + +# sqb login and logout + +> Authorize a bounded CLI session through your server's browser sign-in. + +Online: https://sqlbuild.com/docs/cli/login/ + +Sign in to a SQLBuild server using its configured password, SSO or organization authentication: + +```bash +sqb login --server https://control.example.com +``` + +Only authorize a request you started yourself. Compare the browser's verification code with +your terminal, sign in, and choose **Authorize CLI**. Return to the terminal for the completion +message. Choose **Cancel** to refuse the request. + +The browser confirmation expires after five minutes and can be exchanged once. The CLI receives +its own bounded person session, not your browser cookie or a personal API token. Your existing +permissions still apply; signing in does not create a person, assign a role or grant access. + +## Use a browser on another device + +```bash +sqb login --server https://control.example.com --no-browser +``` + +Open the printed URL in your browser and compare its verification code with the terminal. +Keep the terminal running until it confirms completion. If the request expires or is cancelled, +start `sqb login` again. + +## Check your saved session + +```bash +sqb login status --server https://control.example.com +``` + +This command verifies the saved session with the live server and prints only your person and +session identity as JSON. Disabled people, revoked sessions and password changes are enforced +by the server. If the session is unavailable, sign in again. + +## Sign out + +```bash +sqb logout --server https://control.example.com +``` + +Logout revokes the current CLI session and removes its local credential. It does not sign your +browser out. If the server cannot be reached, local cleanup still runs and the command reports +that remote revocation could not be confirmed. + +Credentials use your OS credential store. On headless Linux without a desktop session, an +unavailable credential service permits a private fallback under +`~/.config/sqlbuild/credentials`, restricted to your user. A locked or ambiguous OS credential +store is an error: unlock it and retry. Credentials are never printed or written into a project. + +| Option | Purpose | +| --- | --- | +| `--server HTTPS_ORIGIN` | Required server origin, without a path, query or embedded credentials | +| `--ca-cert PEM` | Trust an additional CA certificate for this server; certificate and hostname verification remain enabled | +| `--no-browser` | Print the confirmation URL without opening a browser; applies to sign-in | + +Lifecycle messages go to stderr. Successful login and logout leave stdout empty; status writes +identity JSON to stdout. A failure exits nonzero and never prints a credential. diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index f26ad2ab8f..c304771104 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -275,3 +275,17 @@ class ProjectionReplayTestCase: description: str open_store: Callable[[tuple[Path, Path]], AbstractContextManager[ProjectionStore]] expected_progress_facts: int + + +@dataclass(frozen=True) +class CliLoginTestCase: + description: str + authentication: str + expected_person_name: str + + +@dataclass(frozen=True) +class CliCleanupTestCase: + description: str + server: str + expected_exit: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py new file mode 100644 index 0000000000..b6d3773ff7 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py @@ -0,0 +1,351 @@ +"""A real verified HTTPS boundary in front of the installed native control server.""" + +import base64 +import hashlib +import http.client +import json +import ssl +import stat +import subprocess +import threading +import time +import uuid +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from http.cookiejar import CookieJar +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from urllib.parse import SplitResult, parse_qs, urlencode, urlsplit +from urllib.request import HTTPCookieProcessor, HTTPSHandler, Request, build_opener + + +@contextmanager +def gateway(root: Path) -> Iterator[tuple[str, Path, dict[str, str]]]: + """Generate disposable keys; forward HTTP unchanged except hop-by-hop transport headers.""" + certificate: Path = root / "localhost.pem" + key: Path = root / "localhost-key.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-addext", + "basicConstraints=critical,CA:FALSE", + "-keyout", + str(key), + "-out", + str(certificate), + ], + check=True, + capture_output=True, + timeout=15, + ) + state: dict[str, str] = {} + identity_provider: Callable[[BaseHTTPRequestHandler, bytes], None] = provider(key) + + class Forward(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.forward() + + def do_POST(self) -> None: + self.forward() + + def log_message(self, format: str, *args: Any) -> None: + """Do not log handoff or authentication request paths.""" + + def forward(self) -> None: + body: bytes = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if self.path.startswith("/issuer/"): + identity_provider(self, body) + return + upstream: http.client.HTTPConnection = http.client.HTTPConnection( + state["backend"], timeout=10 + ) + try: + headers: dict[str, str] = dict(self.headers) + if "proxy-subject" in state: + headers = { + name: value + for name, value in headers.items() + if name.lower() != "x-sqb-user" + } + headers["X-SQB-User"] = state["proxy-subject"] + upstream.request(self.command, self.path, body, headers) + response: http.client.HTTPResponse = upstream.getresponse() + content: bytes = response.read() + self.send_response(response.status) + for name, value in response.getheaders(): + if name.lower() not in {"connection", "transfer-encoding", "content-length"}: + self.send_header(name, value) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + finally: + upstream.close() + + with ThreadingHTTPServer(("127.0.0.1", 0), Forward) as server: + context: ssl.SSLContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certificate, key) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread: threading.Thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"https://localhost:{server.server_port}", certificate, state + finally: + server.shutdown() + thread.join(timeout=10) + + +def browser(origin: str, certificate: Path) -> tuple[Any, CookieJar]: + """Use a real TLS verifier and secure cookie jar, not an insecure browser bypass.""" + cookies: CookieJar = CookieJar() + return build_opener( + HTTPSHandler(context=ssl.create_default_context(cafile=str(certificate))), + HTTPCookieProcessor(cookies), + ), cookies + + +def request( + client: Any, endpoint: str, body: dict[str, Any] | None = None, *, csrf: str = "" +) -> tuple[int, bytes]: + """Return denials without printing authentication response payloads.""" + origin: str = endpoint.split("/", 3)[0] + "//" + endpoint.split("/", 3)[2] + call: Request = Request( + endpoint, + data=None if body is None else json.dumps(body).encode(), + headers={"Origin": origin, "Content-Type": "application/json", "x-sqb-csrf": csrf}, + ) + try: + response: Any = client.open(call, timeout=10) + except HTTPError as error: + response = error + with response: + status: int | None = response.status + assert status is not None + return status, response.read() + + +def wait_for_handoff(process: subprocess.Popen[bytes], log: Path) -> str: + """Observe the CLI's public URL while keeping all credential responses out of logs.""" + deadline: float = time.monotonic() + 20 + while "\nVerification code: " not in log.read_text(): + assert process.poll() is None, "CLI exited before offering browser confirmation" + assert time.monotonic() < deadline, "CLI did not offer browser confirmation" + time.sleep(0.02) + return log.read_text().split("Open ", 1)[1].splitlines()[0] + + +@contextmanager +def pending_login( + command: list[str], environment: dict[str, str], project: Path, log: Path +) -> Iterator[tuple[subprocess.Popen[bytes], str]]: + """Reap a real login process even if browser approval or an assertion fails.""" + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + command, + cwd=project, + env=environment, + stdout=subprocess.PIPE, + stderr=stderr, + ) + try: + yield process, wait_for_handoff(process, log) + finally: + process.kill() + process.wait(timeout=10) + + +def authentication_arguments( + mode: str, origin: str, certificate: Path, principal: str +) -> tuple[str, ...]: + """Use only the product's existing explicit authentication configuration.""" + if mode == "password": + return "--password-origin", origin + configuration: dict[str, Any] = {"origin": origin, "identities": {"orders-person": principal}} + if mode == "oidc": + configuration.update( + issuer=origin + "/issuer", + client_id="orders-control", + certificate_authority=str(certificate), + ) + flag = "--oidc-config" + else: + assert mode == "proxy" + configuration["trusted_peers"] = ["127.0.0.1"] + flag = "--trusted-proxy-config" + path: Path = certificate.parent / "authentication.json" + path.write_text(json.dumps(configuration)) + return flag, str(path) + + +def sign_in(client: Any, origin: str, mode: str, username: str, password: str) -> str: + """Password, PKCE provider, and trusted-proxy authentication all use real installed routes.""" + if mode == "password": + status, payload = request( + client, + origin + "/api/auth/password/login", + {"username": username, "password": password}, + ) + else: + path: str = "/api/auth/oidc/login" if mode == "oidc" else "/api/auth/proxy" + status, payload = request(client, origin + path) + assert status == 200, f"Installed browser authentication failed with HTTP {status}" + return json.loads(payload)["csrf_token"] + + +def assert_private_storage( + credential: Path, command: list[str], environment: dict[str, str], project: Path +) -> None: + """Real CLI reload must refuse unsafe file/ancestor modes and symlink credentials.""" + for directory in [credential, credential.parents[2]]: + original_mode: int = stat.S_IMODE(directory.stat().st_mode) + directory.chmod(0o777) + unsafe: subprocess.CompletedProcess[str] = subprocess.run( + command, + cwd=project, + env=environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert unsafe.returncode != 0 and not unsafe.stdout + directory.chmod(original_mode) + backup: Path = credential.with_suffix(".backup") + credential.rename(backup) + credential.symlink_to(backup) + symlink: subprocess.CompletedProcess[str] = subprocess.run( + command, + cwd=project, + env=environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert symlink.returncode != 0 and not symlink.stdout + credential.unlink() + backup.rename(credential) + + +def encoded(value: bytes) -> str: + """JOSE uses unpadded URL-safe base64.""" + return base64.urlsafe_b64encode(value).rstrip(b"=").decode() + + +def provider(key: Path) -> Callable[[BaseHTTPRequestHandler, bytes], None]: + """Consume codes once and validate real PKCE before signing the requested nonce.""" + modulus: str = ( + subprocess.run( + ["openssl", "rsa", "-in", str(key), "-modulus", "-noout"], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + .stdout.strip() + .split("=", 1)[1] + ) + jwk: dict[str, str] = { + "kty": "RSA", + "use": "sig", + "alg": "RS256", + "kid": "orders", + "n": encoded(bytes.fromhex(modulus)), + "e": "AQAB", + } + codes: dict[str, dict[str, list[str]]] = {} + + def handle(handler: BaseHTTPRequestHandler, body: bytes) -> None: + assert isinstance(handler.server, ThreadingHTTPServer) + origin: str = f"https://localhost:{handler.server.server_port}" + issuer: str = origin + "/issuer" + path: SplitResult = urlsplit(handler.path) + payload: dict[str, object] + if path.path.endswith("/.well-known/openid-configuration"): + payload = { + "issuer": issuer, + "authorization_endpoint": issuer + "/authorize", + "token_endpoint": issuer + "/token", + "jwks_uri": issuer + "/jwks", + "response_types_supported": ["code"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + "code_challenge_methods_supported": ["S256"], + } + elif path.path.endswith("/jwks"): + payload = {"keys": [jwk]} + elif path.path.endswith("/authorize"): + parameters: dict[str, list[str]] = parse_qs(path.query) + assert parameters["client_id"] == ["orders-control"] + assert parameters["code_challenge_method"] == ["S256"] + assert parameters["redirect_uri"] == [origin + "/api/auth/oidc/callback"] + code: str = str(uuid.uuid4()) + codes[code] = parameters + handler.send_response(303) + handler.send_header( + "Location", + parameters["redirect_uri"][0] + + "?" + + urlencode({"code": code, "state": parameters["state"][0]}), + ) + handler.send_header("Content-Length", "0") + handler.end_headers() + return + elif path.path.endswith("/token"): + form: dict[str, list[str]] = parse_qs(body.decode()) + parameters = codes.pop(form["code"][0]) + assert form["grant_type"] == ["authorization_code"] + assert form["redirect_uri"] == parameters["redirect_uri"] + assert ( + encoded(hashlib.sha256(form["code_verifier"][0].encode()).digest()) + == parameters["code_challenge"][0] + ) + header: str = encoded(json.dumps({"alg": "RS256", "kid": "orders"}).encode()) + claims: str = encoded( + json.dumps( + { + "iss": issuer, + "sub": "orders-person", + "aud": "orders-control", + "iat": int(time.time()), + "exp": int(time.time()) + 300, + "nonce": parameters["nonce"][0], + } + ).encode() + ) + signing_input: bytes = f"{header}.{claims}".encode() + signature: bytes = subprocess.run( + ["openssl", "dgst", "-sha256", "-sign", str(key)], + input=signing_input, + check=True, + capture_output=True, + timeout=10, + ).stdout + payload = { + "access_token": "synthetic-orders-access", + "token_type": "Bearer", + "expires_in": 300, + "id_token": f"{header}.{claims}.{encoded(signature)}", + } + else: + raise AssertionError("unexpected identity-provider endpoint") + content: bytes = json.dumps(payload).encode() + handler.send_response(200) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(content))) + handler.end_headers() + handler.wfile.write(content) + + return handle diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py new file mode 100644 index 0000000000..ad1c6477ca --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py @@ -0,0 +1,297 @@ +"""Installed login persists a bounded person session, reloads it, and observes live revocation.""" + +import json +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CliCleanupTestCase, + CliLoginTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.login.helpers import ( + assert_private_storage, + authentication_arguments, + browser, + gateway, + pending_login, + request, + sign_in, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + environment, + serve, + successful, +) + + +@pytest.mark.skipif( + sys.platform != "linux", reason="Linux owner-only fallback acceptance requires a Linux runtime" +) +@pytest.mark.parametrize( + "test_case", + [ + CliLoginTestCase( + "installed HTTPS password handoff and credential lifecycle", "password", "orders-reader" + ), + CliLoginTestCase( + "installed HTTPS OIDC handoff and credential lifecycle", "oidc", "orders-reader" + ), + CliLoginTestCase( + "installed HTTPS proxy handoff and credential lifecycle", "proxy", "orders-reader" + ), + ], + ids=lambda case: case.description, +) +def test_given_installed_browser_login_when_reloaded_revoked_and_logged_out_then_person_session_and_storage_boundaries_hold( + test_case: CliLoginTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + home: Path = tmp_path / "person-home" + home.mkdir() + project: Path = tmp_path / "empty-project" + project.mkdir() + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/cli_login.json").read_text() + ) + successful(administer(installation, ["create-user", "orders-admin"])) + person: dict[str, Any] = successful( + administer(installation, ["create-principal", test_case.expected_person_name]) + ) + successful( + administer(installation, ["set-password", person["principal_id"]], fixture["password"]) + ) + client_environment: dict[str, str] = environment({"HOME": str(home)}) + client_environment.pop("DBUS_SESSION_BUS_ADDRESS", None) + client_environment.pop("XDG_RUNTIME_DIR", None) + with gateway(tmp_path) as (origin, certificate, transport): + transport.update({"proxy-subject": "orders-person"}) + arguments: tuple[str, ...] = authentication_arguments( + test_case.authentication, origin, certificate, person["principal_id"] + ) + with serve(installation, authentication_arguments=arguments) as backend: + transport["backend"] = backend.removeprefix("http://") + client, cookies = browser(origin, certificate) + csrf: str = sign_in( + client, origin, test_case.authentication, person["name"], fixture["password"] + ) + assert request(client, origin + "/api/control/audit")[0] == 403 + common: list[str] = ["--server", origin, "--ca-cert", str(certificate)] + log: Path = tmp_path / "cli-login.log" + with pending_login( + [binary, "login", *common, "--no-browser"], client_environment, project, log + ) as (process, url): + assert url.startswith(f"{origin}/auth/cli#") + status, payload = request(client, url.split("#")[0]) + assert status == 200 and b"Authorize CLI" in payload + status, payload = request( + client, + f"{origin}/api/auth/cli/approve", + { + "browser_code": url.split("#", 1)[1], + "approve": True, + }, + csrf=csrf, + ) + assert status == 204 and payload == b"" + stdout, _ = process.communicate(timeout=20) + assert process.returncode == 0, "CLI handoff did not complete" + assert not stdout and "signed in successfully" in log.read_text() + assert not list(project.iterdir()) + credential_files: list[Path] = list( + (home / ".config/sqlbuild/credentials").glob("*.json") + ) + assert len(credential_files) == 1 + assert stat.S_IMODE(credential_files[0].stat().st_mode) == 0o600 + assert stat.S_IMODE(credential_files[0].parent.stat().st_mode) == 0o700 + saved: dict[str, Any] = json.loads(credential_files[0].read_text()) + assert ( + saved["bearer"] not in log.read_text() + and saved["csrf_token"] not in log.read_text() + ) + assert ( + saved["identity"]["session_id"] + != json.loads(request(client, f"{origin}/api/auth/session")[1])["session_id"] + ) + checked: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "login", "status", *common], + cwd=project, + env=client_environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert checked.returncode == 0 + assert json.loads(checked.stdout)["principal_id"] == person["principal_id"] + assert "bearer" not in checked.stdout and "csrf_token" not in checked.stdout + assert_private_storage( + credential_files[0], + [binary, "login", "status", *common], + client_environment, + project, + ) + successful( + administer( + installation, ["set-password", person["principal_id"]], fixture["replacement"] + ) + ) + revoked: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "login", "status", *common], + cwd=project, + env=client_environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert revoked.returncode != 0 and not revoked.stdout and "revoked" in revoked.stderr + for logout_arguments, expected_exit in [ + (["--server", origin, "--ca-cert", str(tmp_path / "missing-ca.pem")], 1), + (common, 0), + ]: + logout: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "logout", *logout_arguments], + cwd=project, + env=client_environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert logout.returncode == expected_exit and not logout.stdout + assert not list(credential_files[0].parent.iterdir()) + assert not list(project.iterdir()) + assert len(list(cookies)) > 0 + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux fallback path regression") +@pytest.mark.parametrize( + "test_case", + [ + CliCleanupTestCase( + "absent fallback permits HTTPS logout", "https://control.example.com", 0 + ), + CliCleanupTestCase("plaintext origin is still refused", "http://control.example.com", 1), + ], + ids=lambda case: case.description, +) +def test_given_absent_fallback_when_configuration_is_symlinked_then_logout_does_not_create_or_validate_unused_storage( + test_case: CliCleanupTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + home: Path = tmp_path / "home" + home.mkdir() + configuration: Path = tmp_path / "configuration" + configuration.mkdir() + (home / ".config").symlink_to(configuration, target_is_directory=True) + values: dict[str, str] = environment({"HOME": str(home)}) + values.pop("DBUS_SESSION_BUS_ADDRESS", None) + values.pop("XDG_RUNTIME_DIR", None) + logout: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "logout", "--server", test_case.server], + cwd=tmp_path, + env=values, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert logout.returncode == test_case.expected_exit and not logout.stdout + assert not list(configuration.iterdir()) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux fallback permission failure") +@pytest.mark.parametrize( + "test_case", + [ + CliLoginTestCase( + "readable credential still revokes remotely when local unlink fails", + "password", + "orders-reader", + ) + ], + ids=lambda case: case.description, +) +def test_given_live_cli_session_when_local_unlink_fails_then_logout_still_revokes_remotely( + test_case: CliLoginTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + home: Path = tmp_path / "home" + home.mkdir() + values: dict[str, str] = environment({"HOME": str(home)}) + values.pop("DBUS_SESSION_BUS_ADDRESS", None) + values.pop("XDG_RUNTIME_DIR", None) + successful(administer(installation, ["create-user", "orders-admin"])) + person: dict[str, Any] = successful( + administer(installation, ["create-principal", test_case.expected_person_name]) + ) + successful( + administer( + installation, ["set-password", person["principal_id"]], "orders-private-password" + ) + ) + with gateway(tmp_path) as (origin, certificate, transport): + arguments: tuple[str, ...] = authentication_arguments( + test_case.authentication, origin, certificate, person["principal_id"] + ) + with serve(installation, authentication_arguments=arguments) as backend: + transport["backend"] = backend.removeprefix("http://") + browser_client, _ = browser(origin, certificate) + csrf: str = sign_in( + browser_client, + origin, + test_case.authentication, + person["name"], + "orders-private-password", + ) + common: list[str] = ["--server", origin, "--ca-cert", str(certificate)] + with pending_login( + [binary, "login", *common, "--no-browser"], values, tmp_path, tmp_path / "cli.log" + ) as (process, url): + assert ( + request( + browser_client, + origin + "/api/auth/cli/approve", + {"browser_code": url.split("#", 1)[1], "approve": True}, + csrf=csrf, + )[0] + == 204 + ) + process.communicate(timeout=20) + assert process.returncode == 0 + credential: Path = next((home / ".config/sqlbuild/credentials").glob("*.json")) + saved: dict[str, Any] = json.loads(credential.read_text()) + cli_client, _ = browser(origin, certificate) + cli_client.addheaders = [("Authorization", f"Session {saved['bearer']}")] + assert request(cli_client, origin + "/api/auth/session")[0] == 200 + credential.parent.chmod(0o500) + try: + logout: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "logout", *common], + cwd=tmp_path, + env=values, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert logout.returncode != 0 and not logout.stdout + assert credential.exists() + assert request(cli_client, origin + "/api/auth/session")[0] == 401 + finally: + credential.parent.chmod(0o700) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py index 11785d7dae..45c0911a60 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py @@ -47,7 +47,13 @@ def successful(output: subprocess.CompletedProcess[str]) -> dict[str, Any]: @contextmanager def serve( - installation: tuple[Path, Path], *, extra_environment: dict[str, str] | None = None + installation: tuple[Path, Path], + *, + extra_environment: dict[str, str] | None = None, + authentication_arguments: tuple[str, ...] = ( + "--password-origin", + "https://control.example.com", + ), ) -> Iterator[str]: """Kill and reap the server before returning, on Windows as well as Unix.""" binary, root = installation @@ -60,8 +66,7 @@ def serve( "--port", "0", "--no-scheduler", - "--password-origin", - "https://control.example.com", + *authentication_arguments, ], cwd=root, env=environment(extra_environment), diff --git a/website/src/content/docs/docs/cli/login.mdx b/website/src/content/docs/docs/cli/login.mdx new file mode 100644 index 0000000000..f00d73ceb3 --- /dev/null +++ b/website/src/content/docs/docs/cli/login.mdx @@ -0,0 +1,64 @@ +--- +title: "sqb login and logout" +description: "Authorize a bounded CLI session through your server's browser sign-in." +sidebar: + label: "login / logout" +--- + +Sign in to a SQLBuild server using its configured password, SSO or organization authentication: + +```bash +sqb login --server https://control.example.com +``` + +Only authorize a request you started yourself. Compare the browser's verification code with +your terminal, sign in, and choose **Authorize CLI**. Return to the terminal for the completion +message. Choose **Cancel** to refuse the request. + +The browser confirmation expires after five minutes and can be exchanged once. The CLI receives +its own bounded person session, not your browser cookie or a personal API token. Your existing +permissions still apply; signing in does not create a person, assign a role or grant access. + +## Use a browser on another device + +```bash +sqb login --server https://control.example.com --no-browser +``` + +Open the printed URL in your browser and compare its verification code with the terminal. +Keep the terminal running until it confirms completion. If the request expires or is cancelled, +start `sqb login` again. + +## Check your saved session + +```bash +sqb login status --server https://control.example.com +``` + +This command verifies the saved session with the live server and prints only your person and +session identity as JSON. Disabled people, revoked sessions and password changes are enforced +by the server. If the session is unavailable, sign in again. + +## Sign out + +```bash +sqb logout --server https://control.example.com +``` + +Logout revokes the current CLI session and removes its local credential. It does not sign your +browser out. If the server cannot be reached, local cleanup still runs and the command reports +that remote revocation could not be confirmed. + +Credentials use your OS credential store. On headless Linux without a desktop session, an +unavailable credential service permits a private fallback under +`~/.config/sqlbuild/credentials`, restricted to your user. A locked or ambiguous OS credential +store is an error: unlock it and retry. Credentials are never printed or written into a project. + +| Option | Purpose | +| --- | --- | +| `--server HTTPS_ORIGIN` | Required server origin, without a path, query or embedded credentials | +| `--ca-cert PEM` | Trust an additional CA certificate for this server; certificate and hostname verification remain enabled | +| `--no-browser` | Print the confirmation URL without opening a browser; applies to sign-in | + +Lifecycle messages go to stderr. Successful login and logout leave stdout empty; status writes +identity JSON to stdout. A failure exits nonzero and never prints a credential. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 2be1e116b7..8e33f35aac 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -201,6 +201,7 @@ "docs/cli/playground", "docs/cli/skills", "docs/cli/server", + "docs/cli/login", "docs/cli/compile", "docs/cli/format", "docs/cli/contract", From ec29d2f34b4416daefc311d720a204820bf00dd3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:04:45 +0000 Subject: [PATCH 119/138] test: retain populated concurrency slots during repeated recovery --- .../edge_cases/orchestration/local_recovery_orders.py | 11 +++++++++++ .../publication/test_local_recovery.py | 9 +++++++++ 2 files changed, 20 insertions(+) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py new file mode 100644 index 0000000000..92f64e9d81 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py @@ -0,0 +1,11 @@ +"""Keep a populated concurrency slot until recovered terminal publication commits.""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build orders under a shared concurrency limit.", + select="orders", + concurrency_groups=("orders_limit",), + schedule=schedule(cron="0 0 1 1 *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index a9533cdef3..c477f19e21 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -110,6 +110,12 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an "failed", [], launcher="uncooperative", + concurrency_limits={"orders_limit": 1}, + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py" + ).read_text(), + }, store_setup=Path( "benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql" ).read_text(), @@ -128,6 +134,9 @@ def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_w assert active["status"] == "launched" tables: tuple[str, ...] = ("sqb_write_claims", "sqb_concurrency_slots", "sqb_run_liveness") with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT group_key, run_id FROM sqb_concurrency_slots" + ).fetchall() == [("orders_limit", active["run_id"])] retained: dict[str, list[tuple[Any, ...]]] = { table: control.execute(f"SELECT * FROM {table}").fetchall() for table in tables } From 19b01dccbef2576012d4c92803fc1af568f1992a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:34:03 +0000 Subject: [PATCH 120/138] feat: expose scoped published SQL inspection --- .../administrator_lifecycle.json | 10 +- .../edge_cases/orchestration/node_sql.json | 7 + clients/typescript/src/schema.d.ts | 229 ++++++++++++++++++ .../authentication/password_sessions.rs | 4 + .../src/api/_helpers/resources/mod.rs | 1 + .../api/_helpers/resources/person_node_sql.rs | 32 +++ .../sqlbuild-orchestration/src/api/models.rs | 4 +- .../src/snapshots/main/mod.rs | 1 + .../snapshots/main/read_person_node_sql.rs | 66 +++++ .../src/snapshots/models.rs | 9 + .../tests/api/node_sql/helpers.rs | 130 ++++++++++ .../tests/api/person_sessions/test_types.rs | 1 + .../tests/api/test_contracts.rs | 2 + .../tests/api/test_person_sessions.rs | 100 ++++++++ .../sqlbuild/references/docs/cli/server.md | 16 ++ .../publication/person/helpers.py | 40 +++ .../publication/test_person_journey.py | 8 + website/src/content/docs/docs/cli/server.mdx | 16 ++ 18 files changed, 670 insertions(+), 6 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_sql.json create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json index 93cafde23a..e002f31cc4 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -13,11 +13,11 @@ {"role": "developer", "allowed": true} ], "session_roles": [ - {"role": null, "expected_metadata": 403, "expected_connection": 403, "expected_job": 403}, - {"role": "viewer", "expected_metadata": 200, "expected_connection": 403, "expected_job": 403}, - {"role": "operator", "expected_metadata": 200, "expected_connection": 403, "expected_job": 200}, - {"role": "developer", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200}, - {"role": "admin", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200} + {"role": null, "expected_metadata": 403, "expected_connection": 403, "expected_job": 403, "expected_sql": 403}, + {"role": "viewer", "expected_metadata": 200, "expected_connection": 403, "expected_job": 403, "expected_sql": 403}, + {"role": "operator", "expected_metadata": 200, "expected_connection": 403, "expected_job": 200, "expected_sql": 403}, + {"role": "developer", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200, "expected_sql": 200}, + {"role": "admin", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200, "expected_sql": 200} ], "job_settings": [ {"full_refresh": false}, diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json new file mode 100644 index 0000000000..1e8b77124a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json @@ -0,0 +1,7 @@ +{ + "raw_sql": "SELECT {{ order_amount() }} AS amount -- Orders 🧾\n", + "compiled_sql": "SELECT 23 AS amount -- Orders 🧾\n", + "python_source": "def order_amount():\n return 23\n", + "function_sql": "43", + "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 5d3f3d8bc6..ffc7f2b06f 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -32,6 +32,54 @@ export interface paths { patch?: never; trace?: never; }; + "/api/auth/cli/approve": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["approveCliLogin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/cli/claim": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["claimCliLogin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/cli/start": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["startCliLogin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/auth/logout": { parameters: { query?: never; @@ -243,6 +291,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonNodeSql"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -872,6 +936,23 @@ export interface components { ChangeNotice: { topic: string; }; + CliLoginApprovalRequest: { + approve: boolean; + browser_code: string; + }; + CliLoginClaimRequest: { + client_secret: string; + /** Format: uuid */ + flow_id: string; + }; + CliLoginSessionResponse: { + bearer: string; + csrf_token: string; + identity: components["schemas"]["SessionIdentity"]; + }; + CliLoginStartRequest: { + client_hash: string; + }; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -1127,6 +1208,14 @@ export interface components { NodeEventKind: "success" | "failed" | "recovered"; /** @enum {string} */ NodeOutcome: "success" | "warn" | "failed" | "skipped" | "cancelled"; + /** @description SQL inspection never returns Python source, execution artifacts or unrelated node metadata. */ + NodeSql: { + compiled_sql?: string | null; + node_key: string; + raw_sql?: string | null; + /** Format: uuid */ + snapshot_id: string; + }; NodeStatus: { build?: null | components["schemas"]["FingerprintObservation"]; definition: components["schemas"]["DefinitionEvidence"]; @@ -1595,6 +1684,11 @@ export interface components { /** Format: uuid */ session_id: string; }; + StartedLogin: { + browser_code: string; + /** Format: date-time */ + expires_at: string; + }; StatisticsBatch: { /** Format: int64 */ generation: number; @@ -2086,6 +2180,103 @@ export interface operations { }; }; }; + approveCliLogin: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CliLoginApprovalRequest"]; + }; + }; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + claimCliLogin: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CliLoginClaimRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CliLoginSessionResponse"]; + }; + }; + 202: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + startCliLogin: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CliLoginStartRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StartedLogin"]; + }; + }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; logoutSession: { parameters: { query?: never; @@ -2597,6 +2788,44 @@ export interface operations { }; }; }; + readPersonNodeSql: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + node_key: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["NodeSql"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index ce24edb6c1..3fff0bfc61 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -90,6 +90,10 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/control/locations/{location_id}/targets/{target}/node-status", get(crate::api::_helpers::resources::person_operations::node_status), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql", + get(crate::api::_helpers::resources::person_node_sql::read), + ) .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index c85a143786..a66f36cbc8 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod docker_image; pub(crate) mod location_health; pub(crate) mod location_work; pub(crate) mod person_location_work; +pub(crate) mod person_node_sql; pub(crate) mod person_operations; pub(crate) mod router; pub(crate) mod run_reads; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs new file mode 100644 index 0000000000..c60a65951a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs @@ -0,0 +1,32 @@ +//! SQL-only inspection uses live person grants, not the machine snapshot endpoint. + +use axum::extract::{Path, State}; +use axum::http::HeaderMap; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::password_transport::session_credentials; +use crate::api::errors::ApiError; +use crate::credentials::models::AuditContext; +use crate::snapshots::main::read_person_node_sql::read_person_node_sql; +use crate::snapshots::models::NodeSql; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql", operation_id = "readPersonNodeSql", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("node_key" = String, Path)), + responses((status = 200, body = Option), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn read( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target, node_key)): Path<(Uuid, String, String)>, + headers: HeaderMap, +) -> Result>, ApiError> { + Ok(Json( + read_person_node_sql( + &store, + (&context, session_credentials(&headers)), + (location_id, &target, &node_key), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index f48cfde010..ebdda2836d 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -7,7 +7,8 @@ use crate::api::_helpers::authentication::trusted_proxy; use crate::api::_helpers::authentication::{cli_login, oidc}; use crate::api::_helpers::resources::{ administrators, audit, changes, location_health, location_work, person_location_work, - person_operations, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, + person_node_sql, person_operations, run_reads, runs, snapshots, trigger_reads, + warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] @@ -26,6 +27,7 @@ use crate::api::_helpers::resources::{ person_operations::node_status, person_location_work::request, person_location_work::result, + person_node_sql::read, password_sessions::login, password_sessions::current, password_sessions::logout, diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 5f9ca853fc..92e8dd726d 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod publish_development_snapshot; pub mod publish_remote_snapshot; pub mod publish_snapshot; pub(crate) mod read_current_record; +pub mod read_person_node_sql; pub(crate) mod read_snapshot_documents; pub(crate) mod read_snapshot_nodes; pub(crate) mod rebuild_projections; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs new file mode 100644 index 0000000000..366acd4335 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs @@ -0,0 +1,66 @@ +//! Live SQL permission, current publication and sensitive-read audit share one writer fence. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::snapshots::_helpers::reading::attach_sources; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::models::NodeSql; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn read_person_node_sql( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location_id, target, node_key): (Uuid, &str, &str), +) -> Result, ControlError> { + let scope = PermissionScope { + action: Action::ViewSql, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [node_key.into()].into(), + }; + let (transaction, context) = authorize_session( + begin(store, TransactionMode::Write).await?, + credentials, + &scope, + DefaultRole::None, + ) + .await?; + let (mut transaction, record) = read_current_record(transaction, (location_id, target)).await?; + let mut result = None; + if let Some(record) = record { + let nodes; + (transaction, nodes) = + read_snapshot_nodes(transaction, record.snapshot_id, Some(&scope.nodes)).await?; + if !nodes.is_empty() { + let rows; + (transaction, rows) = query(transaction, + "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash \ + WHERE s.snapshot_id=$1 AND s.node_key=$2 AND s.kind IN ('raw_sql','compiled_sql')", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?; + if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) { + result = Some(NodeSql { + snapshot_id: record.snapshot_id, + node_key: node_key.into(), + raw_sql: node.raw_sql, + compiled_sql: node.compiled_sql, + }); + } + } + } + let transaction = append_audit( + transaction, + &context, + (Uuid::now_v7(), "sql.read"), + (json!(scope), json!(null), json!({"snapshot_id":result.as_ref().map(|node| node.snapshot_id),"found":result.is_some()})), + ) + .await?; + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 20d474c674..7d8fa1bc0c 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -17,6 +17,15 @@ pub struct SnapshotNode { pub python_source: Option, } +/// SQL inspection never returns Python source, execution artifacts or unrelated node metadata. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct NodeSql { + pub snapshot_id: Uuid, + pub node_key: String, + pub raw_sql: Option, + pub compiled_sql: Option, +} + /// A location's successful compile for one target. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct SnapshotInput { diff --git a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs new file mode 100644 index 0000000000..9c08fe2846 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs @@ -0,0 +1,130 @@ +//! Sensitive SQL reads disclose only the current exact node after durable audit publication. + +use reqwest::{Client, Method, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::person_sessions_helpers::request; +use crate::api::tests::person_sessions_test_types::BrowserSession; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotInput; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; + +pub(crate) async fn node_sql_boundaries( + store: &ControlStore, + (client, session): (&Client, &BrowserSession), + (location, snapshot_id): (Uuid, Uuid), + (input, data): (&SnapshotInput, &Value), +) { + let path = format!("/api/control/locations/{location}/targets/development/nodes/orders/sql"); + for other in [ + path.replace("development", "preview"), + path.replace(&location.to_string(), &Uuid::now_v7().to_string()), + ] { + assert_eq!( + request(client, session, (Method::GET, &other), (&Value::Null, "")) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + for (key, expected) in [ + ( + "udf:orders", + json!({"snapshot_id":snapshot_id,"node_key":"udf:orders","raw_sql":null,"compiled_sql":data["function_sql"]}), + ), + ("missing-orders", Value::Null), + ("orders+", Value::Null), + ] { + let response = request( + client, + session, + ( + Method::GET, + &path.replace("nodes/orders/", &format!("nodes/{key}/")), + ), + (&Value::Null, ""), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("exact node"), + expected + ); + } + let mut before = logical_state(store).await; + let read = request(client, session, (Method::GET, &path), (&Value::Null, "")).await; + assert_eq!(read.status(), StatusCode::OK); + assert_eq!( + read.json::().await.expect("current SQL"), + json!({"snapshot_id":snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["compiled_sql"]}) + ); + let mut after = logical_state(store).await; + assert_ne!( + before.remove("sqb_security_audit"), + after.remove("sqb_security_audit") + ); + assert_eq!( + before, after, + "SQL inspection changes only the security audit" + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("audit cut"); + let (transaction, audit) = query(transaction, "SELECT actor,action,granting_role,scope_json,before_json,after_json FROM sqb_security_audit WHERE action='sql.read' ORDER BY storage_id DESC LIMIT 1", vec![]).await.expect("audit"); + commit(transaction).await.expect("close audit cut"); + assert_eq!(audit[0]["granting_role"], "developer"); + assert_eq!(audit[0]["scope_json"]["action"], "view.sql"); + assert_eq!(audit[0]["scope_json"]["nodes"], json!(["orders"])); + assert_eq!( + audit[0]["after_json"], + json!({"snapshot_id":snapshot_id,"found":true}) + ); + assert!( + !json!(audit) + .to_string() + .contains(data["raw_sql"].as_str().expect("raw SQL")) + ); + let fault = TransactionFault { + description: "SQL read requires durable audit", + table: "sqb_security_audit", + operation: "INSERT", + condition: "WHEN NEW.action='sql.read'", + sqlite_body: "SELECT RAISE(ABORT, 'synthetic SQL audit failure')", + postgres_body: "RAISE EXCEPTION 'synthetic SQL audit failure'", + }; + install_fault(store, &fault).await; + let before = logical_state(store).await; + let response = request(client, session, (Method::GET, &path), (&Value::Null, "")).await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + response.json::().await.expect("sanitized refusal"), + json!({"error":"control-store operation failed"}) + ); + assert_eq!( + logical_state(store).await, + before, + "audit failure discloses no SQL and changes no rows" + ); + remove_fault(store, &fault).await; + let mut updated = input.clone(); + updated.generation += 1; + updated + .nodes + .get_mut("orders") + .expect("orders") + .compiled_sql = Some(data["updated_sql"].as_str().expect("updated SQL").into()); + let current = publish_snapshot(store, &updated) + .await + .expect("new publication"); + let response = request(client, session, (Method::GET, &path), (&Value::Null, "")).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("latest SQL"), + json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"]}) + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs b/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs index dbef04a914..36a31f7a0f 100644 --- a/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs @@ -15,4 +15,5 @@ pub(crate) struct SessionRoleCase { pub(crate) expected_metadata: u16, pub(crate) expected_connection: u16, pub(crate) expected_job: u16, + pub(crate) expected_sql: u16, } diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index d8de3cff39..c126023cfc 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -6,6 +6,8 @@ mod changes_helpers; mod fingerprint_reads_helpers; #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "node_sql/helpers.rs"] +mod node_sql_helpers; #[path = "node_status/helpers.rs"] mod node_status_helpers; #[path = "oidc/helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index 376616db8e..03979f0fe9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -29,6 +29,7 @@ use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; use crate::sensors::tests::helpers::snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; use crate::store::models::TransactionMode; use crate::store::tests::replica_helpers::second_store; @@ -92,6 +93,43 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ let mut input = snapshot(location); input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = json!({"kind":"custom", "function":"orders_ready", "code_hash":"orders_v1"}); + let sql_input: Value = serde_json::from_str( + &std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/node_sql.json"), + ) + .expect("SQL fixture"), + ) + .expect("SQL inputs"); + let orders = input.nodes.get_mut("orders").expect("orders node"); + orders.raw_sql = Some(sql_input["raw_sql"].as_str().expect("raw SQL").into()); + orders.compiled_sql = Some( + sql_input["compiled_sql"] + .as_str() + .expect("compiled SQL") + .into(), + ); + orders.python_source = Some( + sql_input["python_source"] + .as_str() + .expect("Python source") + .into(), + ); + input.nodes.insert( + "udf:orders".into(), + SnapshotNode { + kind: "udf".into(), + definition: json!({"name":"orders", "arguments":[], "returns":"INTEGER"}), + raw_sql: None, + compiled_sql: Some( + sql_input["function_sql"] + .as_str() + .expect("function SQL") + .into(), + ), + python_source: None, + }, + ); let published = publish_snapshot(store, &input).await.expect("publication"); assert_eq!( published.generation, test_case.expected_generation, @@ -131,6 +169,8 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ let path = format!("/api/control/locations/{location}/targets/development/connection-tests"); let metadata = format!("/api/control/locations/{location}/targets/development/runs"); + let sql_path = + format!("/api/control/locations/{location}/targets/development/nodes/orders/sql"); let inputs: Value = serde_json::from_str(&std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json")).expect("DuckShop matrix")).expect("JSON"); let roles: Vec = serde_json::from_value(inputs["session_roles"].clone()) @@ -196,6 +236,48 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ role_case.expected_metadata == StatusCode::OK.as_u16() ); assert!(!page.to_string().contains("select 1 as id")); + let sql = request( + &provider.client, + session, + (Method::GET, &sql_path), + (&Value::Null, ""), + ) + .await; + assert_eq!(sql.status().as_u16(), role_case.expected_sql); + assert_eq!(sql.headers()["cache-control"], "no-store"); + let sql: Value = sql.json().await.expect("SQL response"); + for (key, expected) in [ + ("snapshot_id", json!(published.snapshot_id)), + ("node_key", json!("orders")), + ("raw_sql", sql_input["raw_sql"].clone()), + ("compiled_sql", sql_input["compiled_sql"].clone()), + ] { + assert_eq!( + sql.get(key), + [None, Some(&expected)][usize::from(role_case.expected_sql == 200)] + ); + } + assert_eq!( + sql.get("decision_id").is_some(), + role_case.expected_sql != 200 + ); + assert!(sql.get("python_source").is_none()); + let unpublished = request( + &provider.client, + session, + (Method::GET, &sql_path.replace("development", "preview")), + (&Value::Null, ""), + ) + .await; + assert_eq!(unpublished.status().as_u16(), role_case.expected_sql); + assert_eq!( + unpublished + .json::() + .await + .expect("unpublished scope") + .is_null(), + role_case.expected_sql == 200 + ); let job = json!({"run_id":Uuid::now_v7(), "location_id":location, "target":"development", "job_key":"orders", "reason":"Refresh orders", "priority":0}); assert_eq!( request( @@ -275,6 +357,17 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ .expect("revoke through independent store"); } for session in &sessions { + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &sql_path), + (&Value::Null, "") + ) + .await + .status(), + StatusCode::FORBIDDEN + ); assert_eq!( request( &provider.client, @@ -334,6 +427,13 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ ) .await .expect("scoped developer"); + crate::api::tests::node_sql_helpers::node_sql_boundaries( + store, + (&provider.client, &sessions[0]), + (location, published.snapshot_id), + (&input, &sql_input), + ) + .await; for session in &sessions { let result = request( &provider.client, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index c3f165ff54..3731a5bb41 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -14,6 +14,22 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Inspect published SQL + +Use the person-authenticated API to inspect one exact node in the current successful publication. +Send a `GET` request to +`/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql`. + +Developer or Admin access must cover the requested location and target. Viewer, Operator and +the configurable default Viewer role do not grant SQL inspection. The response contains the +snapshot ID, exact node key, raw SQL and compiled SQL. Missing SQL fields are `null`; an absent +current publication or node returns JSON `null` after authorization. + +SQL inspection does not execute SQL or return Python source, execution artifacts, logs or sample +data. Every successful inspection records an audit entry without copying SQL into it. If the +audit cannot be saved, the request fails without returning SQL. Revocation takes effect on the +next request, including sessions created by [`sqb login`](login.md). + ## Repair control views Repair snapshot selection, jobs, triggers, runs, node status and the structural catalog from diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 273799561e..c82e4914d2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -12,7 +12,9 @@ from urllib.error import HTTPError from urllib.parse import urlencode from urllib.request import Request, urlopen +from uuid import uuid4 +import duckdb import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( @@ -210,6 +212,44 @@ def verify_security_audit( assert audit_rejected.value.code == 403 +def verify_sql_inspection( + api: SnapshotApi, + sessions: tuple[tuple[str, str], tuple[str, str]], + assignment: dict[str, Any], + model: Path, + snapshot_id: str, +) -> None: + """Inspect actual published SQL under a live scoped role, then revoke that role.""" + administrator, session = sessions + path: str = ( + f"/api/control/locations/{api.location_id}/targets/development/nodes/model:orders/sql" + ) + assignment = assignment | {"role": "developer", "revoke": False} + assert person_request(api, "/api/admin/roles", body=assignment, session=administrator) == { + "applied": True + } + sql: dict[str, Any] = person_read(api, path, session=session) + assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert sql["snapshot_id"] == snapshot_id + assert sql["node_key"] == "model:orders" + assert sql["raw_sql"] == model.read_text() + with duckdb.connect() as inspection: + assert inspection.sql(sql["compiled_sql"]).fetchall() == [(41,)] + for wrong_scope in [ + path.replace("development", "preview"), + path.replace(api.location_id, str(uuid4())), + ]: + with pytest.raises(HTTPError) as denied: + person_read(api, wrong_scope, session=session) + assert denied.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=administrator + ) == {"applied": True} + with pytest.raises(HTTPError) as revoked: + person_read(api, path, session=session) + assert revoked.value.code == 403 + + def person_request( api: SnapshotApi, path: str, *, body: Any, session: tuple[str, str] = ("", "") ) -> Any: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index 552ce5fcef..780abc474f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -33,6 +33,7 @@ provision_person, verify_compute_transport, verify_security_audit, + verify_sql_inspection, wait_for_person_catalog, ) @@ -260,6 +261,13 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi with pytest.raises(HTTPError) as revoked: person_request(worker.api, path, body=body, session=session) assert revoked.value.code == 403 + verify_sql_inspection( + worker.api, + (admin_session, session), + assignment, + model, + recovered["record"]["snapshot_id"], + ) @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 917fa72103..6c23139a93 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -13,6 +13,22 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Inspect published SQL + +Use the person-authenticated API to inspect one exact node in the current successful publication. +Send a `GET` request to +`/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql`. + +Developer or Admin access must cover the requested location and target. Viewer, Operator and +the configurable default Viewer role do not grant SQL inspection. The response contains the +snapshot ID, exact node key, raw SQL and compiled SQL. Missing SQL fields are `null`; an absent +current publication or node returns JSON `null` after authorization. + +SQL inspection does not execute SQL or return Python source, execution artifacts, logs or sample +data. Every successful inspection records an audit entry without copying SQL into it. If the +audit cannot be saved, the request fails without returning SQL. Revocation takes effect on the +next request, including sessions created by [`sqb login`](/docs/cli/login/). + ## Repair control views Repair snapshot selection, jobs, triggers, runs, node status and the structural catalog from From d48424e561f2cac3733c2ece846745f7a96cfaf7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:06:44 +0000 Subject: [PATCH 121/138] feat: inspect published SQL with saved CLI sessions --- .../sqlbuild-cli/src/argv/command_table.json | 18 ++ .../src/launcher/_helpers/dispatch.rs | 6 + .../src/native_commands/main/mod.rs | 1 + .../main/run_person_invocation.rs | 60 ++++++ .../src/person_login/_helpers/transport.rs | 56 +++++ .../sqlbuild-cli/src/person_login/main/mod.rs | 1 + .../src/person_login/main/read_node_sql.rs | 32 +++ .../sqlbuild/references/docs/cli/server.md | 18 +- .../publication/test_person_sql.py | 201 ++++++++++++++++++ website/src/content/docs/docs/cli/server.mdx | 18 +- 10 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs create mode 100644 crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 7f8b9949d1..f9520bcfdb 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,23 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "control", + "parser": { + "prog": "sqb control", + "description": "Read one exact node's current published SQL using your saved sqb login session. Results are JSON; progress goes to stderr. No project or warehouse connection is opened.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "dest": "person_action", "choices": ["sql"], "help": "inspect published raw and compiled SQL", "group": 0}, + {"kind": "store", "dest": "person_node", "help": "exact published node key, for example model:orders (not a selector)", "group": 0}, + {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin used by sqb login", "group": 1}, + {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1}, + {"kind": "store", "options": ["--location-id"], "dest": "person_location", "required": true, "help": "published location UUID", "group": 1}, + {"kind": "store", "options": ["--target"], "dest": "target", "required": true, "help": "exact published target", "group": 1} + ] + } + }, { "name": "login", "parser": { @@ -8729,6 +8746,7 @@ } ], "choices_help": [ + {"dest": "control", "metavar": "control", "help": "inspect published SQL with your saved person session"}, {"dest": "login", "metavar": "login", "help": "sign in through a browser or verify a saved CLI session"}, {"dest": "logout", "metavar": "logout", "help": "revoke the CLI session and remove local credentials"}, { diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index 689607a902..4b4165481c 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -15,6 +15,7 @@ use crate::native_commands::main::run_control_invocation::run_control_invocation use crate::native_commands::main::run_location_invocation::run_location_invocation; use crate::native_commands::main::run_login_invocation::run_login_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; +use crate::native_commands::main::run_person_invocation::run_person_invocation; use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; @@ -52,6 +53,11 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - { return code; } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_person_invocation(namespace) + { + return code; + } if let Ok(NativeParse::Invocation { namespace, .. }) = &native && let Some(code) = run_control_invocation(namespace) { diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 18cbcdfe85..c5fb741b92 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -13,3 +13,4 @@ pub(crate) mod run_control_invocation; pub(crate) mod run_location_invocation; pub(crate) mod run_login_invocation; pub mod run_native_invocation; +pub(crate) mod run_person_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs new file mode 100644 index 0000000000..036e68a8d5 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs @@ -0,0 +1,60 @@ +//! Person control commands use the saved HTTPS session, never privileged database access. + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::text; +use crate::native_commands::_helpers::terminal::write_output; +use crate::person_login::main::read_node_sql::read_node_sql; + +pub(crate) fn run_person_invocation(namespace: &Namespace) -> Option { + if text(namespace, "command").as_deref() != Some("control") { + return None; + } + drop(write_output(&[( + Stream::Stderr, + "Reading published SQL…\n", + )])); + let result = (|| { + let location = text(namespace, "person_location") + .unwrap_or_default() + .parse() + .map_err(|_| std::io::Error::other("--location-id must be a UUID"))?; + let ca = text(namespace, "login_ca").map(std::path::PathBuf::from); + let sql = read_node_sql( + ( + &text(namespace, "login_server").unwrap_or_default(), + ca.as_deref(), + ), + ( + location, + &text(namespace, "target").unwrap_or_default(), + &text(namespace, "person_node").unwrap_or_default(), + ), + )?; + serde_json::to_string(&sql).map_err(std::io::Error::other) + })(); + Some(match result { + Ok(json) => { + let lost = write_output(&[(Stream::Stdout, &format!("{json}\n"))]); + if lost.is_some() { + drop(write_output(&[( + Stream::Stderr, + "SQL inspection failed: could not write JSON to stdout\n", + )])); + 1 + } else { + drop(write_output(&[( + Stream::Stderr, + "Published SQL read successfully\n", + )])); + 0 + } + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("SQL inspection failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs index b62d3e990c..a26b02f33d 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs @@ -4,7 +4,10 @@ use std::path::Path; use std::time::Duration; use reqwest::header::{AUTHORIZATION, HeaderValue}; +use sqlbuild_orchestration::api::models::ApiErrorBody; use sqlbuild_orchestration::credentials::models::SessionIdentity; +use sqlbuild_orchestration::snapshots::models::NodeSql; +use uuid::Uuid; use crate::person_login::constants::REQUEST_SECONDS; use crate::person_login::models::{PersonClient, PersonCredential}; @@ -99,3 +102,56 @@ pub(crate) async fn revoke( )) } } + +pub(crate) async fn node_sql( + client: &PersonClient, + credential: &PersonCredential, + (location, target, node): (Uuid, &str, &str), +) -> Result, std::io::Error> { + let mut url = reqwest::Url::parse(&client.origin).map_err(std::io::Error::other)?; + url.path_segments_mut() + .map_err(|()| std::io::Error::other("invalid server origin"))? + .pop_if_empty() + .extend([ + "api", + "control", + "locations", + &location.to_string(), + "targets", + target, + "nodes", + node, + "sql", + ]); + let response = authorized(client, reqwest::Method::GET, url.path(), credential)? + .send() + .await + .map_err(|_| std::io::Error::other("could not read published SQL from the server"))?; + let status = response.status(); + if !status.is_success() { + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(std::io::Error::other( + "CLI session is unavailable or revoked; run sqb login again", + )); + } + let decision = response + .json::() + .await + .map_err(|_| { + std::io::Error::other(format!( + "SQL inspection rejected (HTTP {}); invalid error response", + status.as_u16() + )) + })? + .decision_id; + return Err(std::io::Error::other(format!( + "SQL inspection rejected (HTTP {}{})", + status.as_u16(), + decision.map_or_else(String::new, |id| format!("; decision {id}")) + ))); + } + response + .json() + .await + .map_err(|_| std::io::Error::other("server returned an invalid SQL inspection result")) +} diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index b64986bbb9..bf1d2af998 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -1 +1,2 @@ +pub(crate) mod read_node_sql; pub(crate) mod run_login; diff --git a/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs b/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs new file mode 100644 index 0000000000..99970998c7 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs @@ -0,0 +1,32 @@ +//! A saved person session can inspect published SQL without opening a project or warehouse. + +use std::path::Path; + +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; +use sqlbuild_orchestration::snapshots::models::NodeSql; +use uuid::Uuid; + +use crate::person_login::_helpers::{credentials, transport}; + +pub(crate) fn read_node_sql( + (server, ca): (&str, Option<&Path>), + (location, target, node): (Uuid, &str, &str), +) -> Result, std::io::Error> { + let origin = parse_origin(server, "CLI").map_err(std::io::Error::other)?; + if target.trim().is_empty() || node.trim().is_empty() { + return Err(std::io::Error::other( + "target and exact node key must not be blank", + )); + } + let credential = credentials::load(&origin)? + .ok_or_else(|| std::io::Error::other("not signed in; run sqb login"))?; + let client = transport::connect(&origin, ca)?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(transport::node_sql( + &client, + &credential, + (location, target, node), + )) +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 3731a5bb41..c846e07522 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -16,8 +16,22 @@ Use `sqb server --help` for service, authentication and launcher options. ## Inspect published SQL -Use the person-authenticated API to inspect one exact node in the current successful publication. -Send a `GET` request to +Sign in with [`sqb login`](login.md), then inspect one exact node in the current +successful publication: + +```bash +sqb control sql model:orders \ + --server https://control.example.com \ + --location-id "$LOCATION_ID" \ + --target development +``` + +The command uses your saved person session and prints JSON. It does not load a local project +or open a warehouse connection. Use `--ca-cert PATH` for an additional trusted PEM certificate. +The node key is exact: `model:orders+` does not expand a selector. Progress and failures go to +stderr; a failed request prints no SQL to stdout. Permission denials include their decision ID. + +The equivalent person-authenticated API is a `GET` request to `/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql`. Developer or Admin access must cover the requested location and target. Viewer, Operator and diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py new file mode 100644 index 0000000000..d7e12791d8 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -0,0 +1,201 @@ +"""An installed CLI inspects actual published SQL through a bounded live person session.""" + +import json +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + scheduled_worker, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.login.helpers import ( + browser, + gateway, + pending_login, + request, + sign_in, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + provision_person, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + environment, +) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Headless Linux credential-store acceptance") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="saved person session, actual SQL, exact key and live revocation", + model_sql="SELECT 23 AS amount, '" + "orders " * 400 + "' AS description", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authority_discloses_sources( + test_case: WorkerExecutionTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + home: Path = tmp_path / "home" + home.mkdir() + empty: Path = tmp_path / "empty" + empty.mkdir() + client_environment: dict[str, str] = environment({"HOME": str(home)}) + client_environment.pop("DBUS_SESSION_BUS_ADDRESS", None) + client_environment.pop("XDG_RUNTIME_DIR", None) + binary: str = str(Path(sys.executable).with_name("sqb")) + invoke: partial[subprocess.CompletedProcess[Any]] = partial( + subprocess.run, + cwd=empty, + env=client_environment, + capture_output=True, + text=True, + check=False, + timeout=20, + ) + with ( + gateway(tmp_path) as (origin, certificate, transport), + scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", origin), + ) as worker, + ): + transport["backend"] = worker.api.url.removeprefix("http://") + _ = worker_result(worker, test_case) + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-person" + ) + client, _ = browser(origin, certificate) + csrf: str = sign_in( + client, origin, "password", person["name"], "orders-person-password-2026" + ) + common: list[str] = ["--server", origin, "--ca-cert", str(certificate)] + command: list[str] = [ + binary, + "control", + "sql", + "model:orders", + *common, + "--location-id", + worker.api.location_id, + "--target", + "development", + ] + absent: subprocess.CompletedProcess[str] = invoke(command) + assert absent.returncode != 0 and not absent.stdout and "not signed in" in absent.stderr + with pending_login( + [binary, "login", *common, "--no-browser"], + client_environment, + empty, + tmp_path / "cli-login.log", + ) as (process, url): + assert ( + request( + client, + origin + "/api/auth/cli/approve", + {"browser_code": url.split("#", 1)[1], "approve": True}, + csrf=csrf, + )[0] + == 204 + ) + stdout, _ = process.communicate(timeout=20) + assert process.returncode == 0 and not stdout + denied: subprocess.CompletedProcess[str] = invoke(command) + assert ( + denied.returncode != 0 and not denied.stdout and "HTTP 403; decision " in denied.stderr + ) + administrator, _ = browser(origin, certificate) + admin_csrf: str = sign_in( + administrator, origin, "password", admin["name"], "orders-person-password-2026" + ) + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "developer", + "location_id": worker.api.location_id, + "target": "development", + } + assert ( + request(administrator, origin + "/api/admin/roles", assignment, csrf=admin_csrf)[0] + == 200 + ) + inspected: subprocess.CompletedProcess[str] = invoke(command) + assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr + sql: dict[str, Any] = json.loads(inspected.stdout) + assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert ( + sql["snapshot_id"] + == current_snapshot(worker.api, "development")["record"]["snapshot_id"] + ) + assert sql["node_key"] == "model:orders" + assert sql["raw_sql"] == (worker.project / "models/orders.sql").read_text() + with duckdb.connect() as verification: + assert verification.sql(sql["compiled_sql"]).fetchall() == [ + (*row, "orders " * 400) for row in test_case.expected_rows + ] + assert 4097 <= len(inspected.stdout.encode()) < 8192 + with Path("/dev/full").open("w") as unavailable: + undelivered: subprocess.CompletedProcess[str] = invoke( + command, + capture_output=False, + stdout=unavailable, + stderr=subprocess.PIPE, + ) + assert undelivered.returncode != 0 and "SQL inspection failed" in undelivered.stderr + assert "Published SQL read successfully" not in undelivered.stderr + for node in ["model:orders+", "model:orders?query=1/#café"]: + missing: subprocess.CompletedProcess[str] = invoke(command[:3] + [node] + command[4:]) + assert missing.returncode == 0 and json.loads(missing.stdout) is None + assert ( + request( + administrator, + origin + "/api/admin/roles", + assignment | {"revoke": True}, + csrf=admin_csrf, + )[0] + == 200 + ) + revoked: subprocess.CompletedProcess[str] = invoke(command) + assert ( + revoked.returncode != 0 + and not revoked.stdout + and "HTTP 403; decision " in revoked.stderr + ) + changed: subprocess.CompletedProcess[str] = administer( + (Path(binary), tmp_path / "control"), + ["set-password", person["principal_id"]], + "orders-replacement-password-2026", + ) + assert changed.returncode == 0 + expired: subprocess.CompletedProcess[str] = invoke(command) + assert expired.returncode != 0 and not expired.stdout and "revoked" in expired.stderr + logout: subprocess.CompletedProcess[str] = invoke([binary, "logout", *common]) + assert logout.returncode == 0 and not logout.stdout + signed_out: subprocess.CompletedProcess[str] = invoke(command) + assert ( + signed_out.returncode != 0 + and not signed_out.stdout + and "not signed in" in signed_out.stderr + ) + assert not list(empty.iterdir()) diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 6c23139a93..c8b333552c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -15,8 +15,22 @@ Use `sqb server --help` for service, authentication and launcher options. ## Inspect published SQL -Use the person-authenticated API to inspect one exact node in the current successful publication. -Send a `GET` request to +Sign in with [`sqb login`](/docs/cli/login/), then inspect one exact node in the current +successful publication: + +```bash +sqb control sql model:orders \ + --server https://control.example.com \ + --location-id "$LOCATION_ID" \ + --target development +``` + +The command uses your saved person session and prints JSON. It does not load a local project +or open a warehouse connection. Use `--ca-cert PATH` for an additional trusted PEM certificate. +The node key is exact: `model:orders+` does not expand a selector. Progress and failures go to +stderr; a failed request prints no SQL to stdout. Permission denials include their decision ID. + +The equivalent person-authenticated API is a `GET` request to `/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql`. Developer or Admin access must cover the requested location and target. Viewer, Operator and From 0888f471fdada9529d8297a3b014d5194790a6f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:23:14 +0000 Subject: [PATCH 122/138] fix: retain compile interrupts swallowed by Python callbacks --- .../src/bindings/_helpers/cli/entry.rs | 10 ++- .../_helpers/native_host/interrupt.rs | 84 +++++++++++++++++++ .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_threads/interruptible_join.rs | 4 +- .../cli/commands/main/entry/_test_types.py | 7 ++ .../cli/commands/main/entry/helpers.py | 17 ++++ .../cli/commands/main/entry/test_main.py | 70 ++++++++++++++++ 7 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index c3cc865b87..d684b4bd04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::{Instant, SystemTime}; -use pyo3::exceptions::PySystemExit; +use pyo3::exceptions::{PyKeyboardInterrupt, PySystemExit}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -33,10 +33,12 @@ use crate::bindings::_helpers::cli::termination::{ CommandTerminated, TERMINATED_MESSAGE, TerminationHandler, handle_sigterm, interrupting_signal, signal_exit_code, terminated, terminated_exit, }; +use crate::bindings::_helpers::compile_command::constants::COMPILE_COMMAND_NAME; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::interrupt::{InterruptHandler, interrupted}; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; @@ -570,6 +572,11 @@ fn observed_dispatch<'py>( ); } let command: String = invocation.command_name(); + let _interrupt: Option = if command == COMPILE_COMMAND_NAME { + InterruptHandler::install(py)? + } else { + None + }; drop(take_failure_cause()); drop(take_cost_status()); let started: Instant = Instant::now(); @@ -629,6 +636,7 @@ fn observed_dispatch<'py>( &invocation.recall, ) { Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + Ok(_) if interrupted() => Err(PyKeyboardInterrupt::new_err(())), dispatched => dispatched, }; match dispatched { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs new file mode 100644 index 0000000000..d10774e0fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs @@ -0,0 +1,84 @@ +//! Python's default compilation Ctrl+C handler, retained across unraisable callbacks. + +use std::cell::Cell; + +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; + +thread_local! { + static INTERRUPTED: Cell = const { Cell::new(false) }; +} +const SIGINT: i64 = 2; + +/// The previous Python handler and the command's replacement, restored on every exit path. +pub(crate) struct InterruptHandler { + previous: Py, + installed: Py, +} + +impl InterruptHandler { + /// Capture only Python's default main-thread handler; leave embedding applications' handlers alone. + pub(crate) fn install(py: Python<'_>) -> PyResult> { + let threading = py.import("threading")?; + if !threading + .call_method0("current_thread")? + .is(threading.call_method0("main_thread")?) + { + return Ok(None); + } + let signal = py.import("signal")?; + let previous = signal.call_method1("getsignal", (SIGINT,))?; + if !previous.is(signal.getattr("default_int_handler")?) { + return Ok(None); + } + let installed = PyCFunction::new_closure( + py, + None, + None, + |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + INTERRUPTED.set(true); + Err(PyKeyboardInterrupt::new_err(())) + }, + )?; + INTERRUPTED.set(false); + signal.call_method1("signal", (SIGINT, &installed))?; + Ok(Some(Self { + previous: previous.unbind(), + installed: installed.into_any().unbind(), + })) + } +} + +impl Drop for InterruptHandler { + fn drop(&mut self) { + Python::attach(|py| { + let restored: PyResult<()> = (|| { + let signal = py.import("signal")?; + if signal + .call_method1("getsignal", (SIGINT,))? + .is(self.installed.bind(py)) + { + signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?; + } + Ok(()) + })(); + drop(restored); + }); + INTERRUPTED.set(false); + } +} + +/// Whether the default handler observed Ctrl+C during the current command. +pub(crate) fn interrupted() -> bool { + INTERRUPTED.get() +} + +/// Check both pending Python signals and a Ctrl+C whose exception a Python callback swallowed. +pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> { + py.check_signals()?; + if interrupted() { + return Err(PyKeyboardInterrupt::new_err(())); + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 230b5bbee2..849f8bf165 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod connections; pub(crate) mod constants; pub(crate) mod header_projection; pub(crate) mod host; +pub(crate) mod interrupt; pub(crate) mod model_lookup; pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs index 2c03b0b48e..29aecc94b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs @@ -5,6 +5,8 @@ use std::time::Duration; use pyo3::prelude::{PyErr, Python}; +use crate::bindings::_helpers::native_host::interrupt::check_signals; + /// How often a join checks for pending signals such as Ctrl+C. const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); @@ -21,7 +23,7 @@ pub(crate) fn join_interruptibly( interrupt: impl FnOnce(), ) -> Joined { while !thread.is_finished() { - if let Err(signal) = py.check_signals() { + if let Err(signal) = check_signals(py) { interrupt(); let _ = py.detach(|| thread.join()); return Joined::Interrupted(signal); diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 3d5ba43db7..1c3e3d91f9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -26,6 +26,13 @@ class CliCaptureOutcomeTestCase: expected_error_type: type[BaseException] | None = None +@dataclass(frozen=True) +class InterruptedCommandTestCase: + description: str + argv: list[str] + expected_error_type: type[BaseException] + + @dataclass(frozen=True) class ProjectCreationRoutingTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 47022aea48..630be312c8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import signal +import weakref from collections.abc import Callable from importlib.machinery import ModuleSpec from pathlib import Path @@ -11,6 +13,7 @@ from sqlbuild._native import ( CliEntrypointHandlers, + CompileCommandRequest, SpecConfigError, loader_destination_parts, ) @@ -45,6 +48,20 @@ def noop_handler(*_a: Any, **_k: Any) -> int: return 0 +def compile_with_callback_interrupt(_request: CompileCommandRequest) -> int: + """Deliver one real SIGINT from a callback where Python cannot propagate its exception.""" + + def value() -> None: + return None + + reference: weakref.ReferenceType[Callable[[], None]] = weakref.ref( + value, lambda _: signal.raise_signal(signal.SIGINT) + ) + del value + assert reference() is None + return 0 + + def extract_json_output_fields(args: tuple[object, ...]) -> tuple[bool, Path | None]: """Return (json_output, json_output_path) from a typed request or positional args.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 52f80dc426..299b15387b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1,9 +1,11 @@ from __future__ import annotations import json +import signal import sys from collections.abc import Callable from pathlib import Path +from typing import Any from unittest.mock import patch import pytest @@ -43,6 +45,7 @@ from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( AuditAggregateRenderingTestCase, CompileJsonErrorRenderingTestCase, + InterruptedCommandTestCase, MainErrorRenderingTestCase, MainTestCase, SkillFreshnessNoticeTestCase, @@ -52,6 +55,7 @@ UnprintableError, build_handlers, build_json_recording_handler, + compile_with_callback_interrupt, invalid_loader_destination_error, ) from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @@ -2522,6 +2526,72 @@ def run_compile(_request: CompileCommandRequest) -> int: assert current_execution_identity() is None +@pytest.mark.parametrize( + "test_case", + ( + InterruptedCommandTestCase( + description="default interrupt survives an unraisable weakref callback", + argv=["compile"], + expected_error_type=KeyboardInterrupt, + ), + ), + ids=lambda case: case.description, +) +def test_given_sigint_in_weakref_callback_when_cli_runs_then_default_interrupt_is_retained( + test_case: InterruptedCommandTestCase, +) -> None: + unraisable: list[type[BaseException]] = [] + previous: Any = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with ( + patch("sys.unraisablehook", side_effect=lambda args: unraisable.append(args.exc_type)), + pytest.raises(test_case.expected_error_type), + ): + _main_with_dependencies( + argv=test_case.argv, + handlers=build_handlers(run_compile=compile_with_callback_interrupt), + ) + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert unraisable == [test_case.expected_error_type] + assert _main_with_dependencies(argv=test_case.argv, handlers=build_handlers()) == 0 + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize( + "test_case", + ( + MainTestCase( + description="embedding interrupt handler remains authoritative", + argv=["compile"], + expected_exit_code=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_embedding_sigint_handler_when_cli_runs_then_handler_remains_authoritative( + test_case: MainTestCase, +) -> None: + received: list[int] = [] + + def custom_handler(number: int, _frame: object) -> None: + received.append(number) + + previous: Any = signal.signal(signal.SIGINT, custom_handler) + try: + assert ( + _main_with_dependencies( + argv=test_case.argv, + handlers=build_handlers(run_compile=compile_with_callback_interrupt), + ) + == test_case.expected_exit_code + ) + assert signal.getsignal(signal.SIGINT) is custom_handler + assert received == [signal.SIGINT] + finally: + signal.signal(signal.SIGINT, previous) + + @pytest.mark.parametrize( "test_case", [ From 96e2bd938bcf718c314a16ac8571923ad1923e41 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:46:34 +0000 Subject: [PATCH 123/138] feat: configure the warehouse statistics retention horizon sqb server --statistics-retention-days N (default 90) sets how long hourly warehouse statistics are kept. Person reads hide samples older than the horizon, and the existing bounded cleanup deletes them with the same cutoff. The value is validated before the control store is opened. --- .../sqlbuild-cli/src/argv/command_table.json | 11 ++ .../src/native_commands/_helpers/decision.rs | 20 +++- .../src/native_commands/_helpers/execution.rs | 2 +- .../src/native_commands/main/_run_control.rs | 6 +- .../_helpers/resources/person_operations.rs | 9 +- .../src/api/main/serve_api.rs | 10 +- .../warehouse_metadata/_helpers/statistics.rs | 19 ++-- .../src/warehouse_metadata/constants.rs | 2 +- .../main/read_person_statistics.rs | 6 +- .../main/run_statistics_retention.rs | 8 +- .../src/warehouse_metadata/models.rs | 34 +++++- .../tests/api/helpers.rs | 2 +- .../api/statistics/eligibility/helpers.rs | 2 +- .../tests/api/statistics/helpers.rs | 10 +- .../tests/api/statistics/retention/helpers.rs | 80 ++++++++++++- .../tests/api/test_catalog_reads.rs | 2 +- .../publication/_test_types.py | 14 +++ .../compiled_project/publication/helpers.py | 42 +++++++ .../publication/test_statistics.py | 107 +++++++++++++++++- 19 files changed, 347 insertions(+), 39 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f9520bcfdb..6c11446234 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1143,6 +1143,17 @@ "help": "ambient metadata access for authenticated people: none (default) or installation-wide viewer; never grants SQL, evidence, execution or administration", "group": 1 }, + { + "kind": "store", + "options": [ + "--statistics-retention-days" + ], + "dest": "control_statistics_retention_days", + "default": 90, + "type": "int", + "help": "days of hourly warehouse statistics to keep; older samples are hidden and then deleted (default: 90)", + "group": 1 + }, { "kind": "store", "options": ["--trusted-proxy-config"], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index a86f0cca60..d67dca4e01 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::warehouse_metadata::models::StatisticsRetention; use uuid::Uuid; use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; @@ -260,9 +261,24 @@ pub(crate) fn control_address( Ok(Some(SocketAddr::new(host, port))) } -pub(crate) fn control_authentication( +/// API settings are validated together before any store is opened. +pub(crate) fn control_api_settings( namespace: &Namespace, -) -> Result { +) -> Result<(ApiAuthentication, StatisticsRetention), ControlError> { + let retention = match namespace.get("control_statistics_retention_days") { + Some(Value::Int(value)) => { + StatisticsRetention::days(value.parse::().map_err(|_| { + ControlError::Invalid( + "--statistics-retention-days must be between 1 and 65535".into(), + ) + })?)? + } + _ => StatisticsRetention::default(), + }; + Ok((control_authentication(namespace)?, retention)) +} + +fn control_authentication(namespace: &Namespace) -> Result { Ok(ApiAuthentication { password_origin: text(namespace, "control_password_origin"), default_role: serde_json::from_value(json!( diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 3e1c4ca854..38deaddb3e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -112,7 +112,7 @@ pub(crate) async fn run_development_roles( (ApiAuthentication { development_scope: Some(scope), ..ApiAuthentication::default() - }, Vec::new()), + }, Vec::new(), Default::default()), (move || { drop(write_output(&[(Stream::Stderr, &format!( "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 184649a83c..2656a38d48 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -8,7 +8,7 @@ use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, control_authentication, flag}; +use crate::native_commands::_helpers::decision::{control_address, control_api_settings, flag}; use crate::native_commands::_helpers::execution::{ open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; @@ -27,7 +27,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let authentication = control_authentication(namespace)?; + let (authentication, retention) = control_api_settings(namespace)?; let store = open_control_store(namespace).await?; let (workers, images) = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); @@ -43,7 +43,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - (authentication, images), + (authentication, images, retention), ( move || { drop(write_output(&[( diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index e7c8adf48d..a29cbd754e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -23,7 +23,7 @@ use crate::warehouse_metadata::main::read_person_node_status::read_person_node_s use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{ CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, - NodeStatusQuery, StatisticsPage, StatisticsQuery, + NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, }; #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics", @@ -37,7 +37,10 @@ use crate::warehouse_metadata::models::{ (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn statistics( (State(store), Extension(context)): (State, Extension), - Extension(default_role): Extension, + (Extension(default_role), Extension(retention)): ( + Extension, + Extension, + ), (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), headers: HeaderMap, ) -> Result, ApiError> { @@ -46,7 +49,7 @@ pub(crate) async fn statistics( &store, (&context, session_credentials(&headers)), (location_id, &target, &options), - default_role, + (default_role, retention), ) .await?, )) diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index d9d0d2029d..e6eacbf989 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -12,11 +12,16 @@ use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::run_statistics_retention::run_statistics_retention; +use crate::warehouse_metadata::models::StatisticsRetention; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, - (authentication, images): (ApiAuthentication, Vec), + (authentication, images, retention): ( + ApiAuthentication, + Vec, + StatisticsRetention, + ), (ready, shutdown): ( impl FnOnce() + Send + 'static, impl Future + Send + 'static, @@ -35,6 +40,7 @@ pub async fn serve_api( let application = router(&store, feed) .merge(authentication_routes) .layer(Extension(std::sync::Arc::new(images))) + .layer(Extension(retention)) .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), @@ -53,7 +59,7 @@ pub async fn serve_api( tokio::select! { result = async { server.await } => { result?; Ok(()) }, result = changes::relay(store.clone(), listener_changes, sender) => result, - result = run_statistics_retention(&store) => result, + result = run_statistics_retention(&store, retention) => result, } } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs index 0e4a885fe4..ea7b1b8453 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use serde_json::json; use uuid::Uuid; @@ -11,10 +11,10 @@ use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; use crate::store::types::Row; use crate::warehouse_metadata::_helpers::ingestion::scoped_event_id; -use crate::warehouse_metadata::constants::{MAX_HISTORY_RECORDS, STATISTICS_RETENTION_DAYS}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::models::{ - StatisticsObservation, StatisticsPage, StatisticsQuery, WarehouseCatalogRecord, - WarehouseReconciliationCursor, + StatisticsObservation, StatisticsPage, StatisticsQuery, StatisticsRetention, + WarehouseCatalogRecord, WarehouseReconciliationCursor, }; pub(crate) fn statistics_event_id( @@ -30,9 +30,12 @@ pub(crate) fn statistics_event_id( } /// Expiration is bounded global cleanup, never a lifecycle status or warehouse write. -pub(crate) async fn prune_statistics(store: &ControlStore) -> Result { +pub(crate) async fn prune_statistics( + store: &ControlStore, + retention: StatisticsRetention, +) -> Result { let transaction = begin(store, TransactionMode::Write).await?; - let cutoff = transaction.now - Duration::days(STATISTICS_RETENTION_DAYS); + let cutoff = retention.cutoff(transaction.now); let (transaction, rows) = query(transaction, "DELETE FROM sqb_warehouse_statistics WHERE event_id IN (SELECT event_id FROM sqb_warehouse_statistics \ WHERE observed_at<$1 ORDER BY observed_at,event_id LIMIT $2) RETURNING event_id", @@ -100,7 +103,7 @@ pub(crate) async fn eligible_names( pub(crate) async fn read_statistics_page( transaction: ControlTransaction, (location_id, target): (Uuid, &str), - options: &StatisticsQuery, + (options, retention): (&StatisticsQuery, StatisticsRetention), ) -> Result<(ControlTransaction, StatisticsPage), ControlError> { if options.schema.trim().is_empty() || options.name.trim().is_empty() @@ -114,7 +117,7 @@ pub(crate) async fn read_statistics_page( .limit .unwrap_or(MAX_HISTORY_RECORDS as u32) .clamp(1, MAX_HISTORY_RECORDS as u32) as usize; - let cutoff = transaction.now - Duration::days(STATISTICS_RETENTION_DAYS); + let cutoff = retention.cutoff(transaction.now); let mut parameters = vec![ Parameter::Id(location_id), Parameter::Text(target.into()), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index c8445bfec6..c52cb938ed 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -2,4 +2,4 @@ pub const MAX_HISTORY_RECORDS: usize = 100; pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; -pub(crate) const STATISTICS_RETENTION_DAYS: i64 = 90; +pub(crate) const DEFAULT_STATISTICS_RETENTION_DAYS: u16 = 90; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs index 55e1fa3464..c1fab94512 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs @@ -8,13 +8,13 @@ use crate::store::main::commit::commit; use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::person_reads::begin_view; use crate::warehouse_metadata::_helpers::statistics::read_statistics_page; -use crate::warehouse_metadata::models::{StatisticsPage, StatisticsQuery}; +use crate::warehouse_metadata::models::{StatisticsPage, StatisticsQuery, StatisticsRetention}; pub async fn read_person_statistics( store: &ControlStore, (context, credentials): (&AuditContext, (&str, &str)), (location_id, target, options): (Uuid, &str, &StatisticsQuery), - default_role: DefaultRole, + (default_role, retention): (DefaultRole, StatisticsRetention), ) -> Result { let transaction = begin_view( store, @@ -29,7 +29,7 @@ pub async fn read_person_statistics( ) .await?; let (transaction, page) = - read_statistics_page(transaction, (location_id, target), options).await?; + read_statistics_page(transaction, (location_id, target), (options, retention)).await?; commit(transaction).await?; Ok(page) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs index 4d24463ce7..e7f813539d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/run_statistics_retention.rs @@ -6,14 +6,18 @@ use crate::store::errors::ControlError; use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::statistics::prune_statistics; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::StatisticsRetention; -pub async fn run_statistics_retention(store: &ControlStore) -> Result<(), ControlError> { +pub async fn run_statistics_retention( + store: &ControlStore, + retention: StatisticsRetention, +) -> Result<(), ControlError> { let mut interval = tokio::time::interval(Duration::from_secs(60)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { interval.tick().await; loop { - match prune_statistics(store).await { + match prune_statistics(store, retention).await { Ok(MAX_HISTORY_RECORDS) => tokio::task::yield_now().await, Ok(_) => break, Err(ControlError::Database(error)) if ControlError::transient_database(&error) => { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 98e8a43faf..8ea8b3988e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -1,9 +1,12 @@ //! Existing warehouse state rows, with their physical state-table provenance. -use chrono::{DateTime, NaiveDate, Utc}; +use chrono::{DateTime, Duration, NaiveDate, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::store::errors::ControlError; +use crate::warehouse_metadata::constants::DEFAULT_STATISTICS_RETENTION_DAYS; + /// Preserve observations, not an inferred latest version, age status or run outcome. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] @@ -707,3 +710,32 @@ pub struct StatisticsPage { pub records: Vec, pub next_after: Option, } + +/// Installation configuration: expired samples are hidden on read before bounded pruning removes them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StatisticsRetention { + days: u16, +} + +impl StatisticsRetention { + pub fn days(days: u16) -> Result { + if days == 0 { + return Err(ControlError::Invalid( + "statistics retention must keep at least one day".into(), + )); + } + Ok(Self { days }) + } + + pub(crate) fn cutoff(self, now: DateTime) -> DateTime { + now - Duration::days(i64::from(self.days)) + } +} + +impl Default for StatisticsRetention { + fn default() -> Self { + Self { + days: DEFAULT_STATISTICS_RETENTION_DAYS, + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index 63c313fca8..23df808669 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -22,7 +22,7 @@ pub(crate) async fn start_api_with_authentication( let task = tokio::spawn(serve_api( store, listener, - (authentication, Vec::new()), + (authentication, Vec::new(), Default::default()), (|| {}, async { let _ = signal.await; }), diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs index 8401242322..12c6deb41a 100644 --- a/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs @@ -60,7 +60,7 @@ pub(crate) async fn statistics_eligibility( store, credentials, (location, "development", &options), - DefaultRole::None, + (DefaultRole::None, Default::default()), ) .await .expect("unknown rows and zero bytes remain distinct"); diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs index 407156e02a..c9ca4aeaa8 100644 --- a/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs @@ -79,7 +79,7 @@ pub(crate) async fn statistics_reads( store, credentials, (location, "development", &options), - DefaultRole::None, + (DefaultRole::None, Default::default()), ) .await .expect("page"); @@ -139,7 +139,7 @@ pub(crate) async fn statistics_reads( store, credentials, (location, "development", &options), - DefaultRole::None + (DefaultRole::None, Default::default()) ) .await .expect("exact scope") @@ -156,7 +156,7 @@ pub(crate) async fn statistics_reads( store, credentials, (location, "development", &options), - DefaultRole::None, + (DefaultRole::None, Default::default()), ) .await .expect("first wins"); @@ -167,7 +167,7 @@ pub(crate) async fn statistics_reads( store, credentials, (scope.0, scope.1, &options), - DefaultRole::None + (DefaultRole::None, Default::default()) ) .await, Err(ControlError::Denied { .. }) @@ -179,7 +179,7 @@ pub(crate) async fn statistics_reads( store, credentials, (location, "development", &options), - DefaultRole::None + (DefaultRole::None, Default::default()) ) .await, Err(ControlError::Invalid(_)) diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs index 4403849958..d4350ee296 100644 --- a/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs @@ -10,7 +10,7 @@ use crate::warehouse_metadata::_helpers::statistics::prune_statistics; use crate::warehouse_metadata::main::{ ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, }; -use crate::warehouse_metadata::models::{StatisticsBatch, StatisticsQuery}; +use crate::warehouse_metadata::models::{StatisticsBatch, StatisticsQuery, StatisticsRetention}; pub(crate) async fn statistics_retention( store: &ControlStore, @@ -52,7 +52,7 @@ pub(crate) async fn statistics_retention( store, credentials, (location, "development", &options), - DefaultRole::None + (DefaultRole::None, Default::default()) ) .await .expect("expired rows hidden before cleanup") @@ -61,7 +61,7 @@ pub(crate) async fn statistics_retention( ); for expected in [100, 100, 5, 0] { assert_eq!( - prune_statistics(store) + prune_statistics(store, StatisticsRetention::default()) .await .expect("global bounded expiration"), expected @@ -77,7 +77,7 @@ pub(crate) async fn statistics_retention( store, credentials, (location, "development", &options), - DefaultRole::None + (DefaultRole::None, Default::default()) ) .await .expect("retained points survive") @@ -85,4 +85,76 @@ pub(crate) async fn statistics_retention( .len(), 100 ); + configured_retention(store, credentials, (location, session_id), data).await; +} + +/// A shorter installation horizon hides and then deletes samples the default would keep. +async fn configured_retention( + store: &ControlStore, + credentials: (&AuditContext, (&str, &str)), + (location, session_id): (Uuid, Uuid), + data: &Value, +) { + assert!(StatisticsRetention::days(0).is_err()); + let horizon = StatisticsRetention::days(5).expect("five-day horizon"); + let records: Vec = [10, 2] + .into_iter() + .map(|age| { + let mut record = data["record"].clone(); + record["name"] = json!("customers_horizon"); + record["observed_at"] = json!(Utc::now() - Duration::days(age)); + record + }) + .collect(); + let batch: StatisticsBatch = + serde_json::from_value(json!({"session_id":session_id,"generation":1,"records":records})) + .expect("horizon batch"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("horizon samples") + .applied + ); + let options = StatisticsQuery { + database: None, + schema: "main".into(), + name: "customers_horizon".into(), + after_observed_at: None, + after_event_id: None, + limit: None, + }; + let visible = async |retention| { + read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, retention), + ) + .await + .expect("horizon read") + .records + .len() + }; + assert_eq!(visible(StatisticsRetention::default()).await, 2); + assert_eq!(visible(horizon).await, 1); + assert_eq!( + prune_statistics(store, StatisticsRetention::default()) + .await + .expect("default keeps both"), + 0 + ); + let pruned = prune_statistics(store, horizon) + .await + .expect("configured expiration"); + assert!( + (2..100).contains(&pruned), + "the shorter horizon expires both the old sample and older retained hours" + ); + assert_eq!( + prune_statistics(store, horizon) + .await + .expect("configured expiration completes"), + 0 + ); + assert_eq!(visible(StatisticsRetention::default()).await, 1); } diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index 007b0a39b8..07b7264348 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -326,7 +326,7 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp limit: None, } ), - DefaultRole::None, + (DefaultRole::None, Default::default()), ) .await, Err(ControlError::Denied { .. }) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index c304771104..005457e7e2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -258,6 +258,20 @@ class StatisticsRecoveryTestCase: expected_catalog_presence: tuple[bool, ...] +@dataclass(frozen=True) +class StatisticsHorizonTestCase: + description: str + server_arguments: tuple[str, ...] + expected_relations: tuple[str, ...] + + +@dataclass(frozen=True) +class StatisticsHorizonRejectionTestCase: + description: str + days: str + expected_error: str + + @dataclass(frozen=True) class DenialMetadataTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 5469c313c8..a53fb60d89 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1006,3 +1006,45 @@ def read_columns(**scope: Any) -> Any: ) finally: blocker.rollback() + + +def run_statistics_retention(directory: Path, arguments: tuple[str, ...]) -> str: + """Run one installed server until its first retention pass, returning its stderr.""" + log: Path = directory / "server.log" + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [ + str(Path(sys.executable).with_name("sqb")), + "server", + "--port", + "0", + "--no-scheduler", + *arguments, + ], + cwd=directory, + env=environment, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + _ = api_address(process, log) + deadline: float = time.monotonic() + 10 + while "sentinel_expired" in statistics_relations(directory): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, "retention did not run at startup" + time.sleep(0.05) + finally: + process.terminate() + _ = process.wait(timeout=10) + return log.read_text() + + +def statistics_relations(directory: Path) -> tuple[str, ...]: + """Read the retained sample relations directly, oldest first.""" + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT relation_name FROM sqb_warehouse_statistics ORDER BY observed_at" + ).fetchall() + return tuple(name for (name,) in rows) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py index 6cf373e8cc..ae788c10b8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py @@ -1,15 +1,21 @@ -"""A failed sample must leave catalog reconciliation able to remove dropped relations.""" +"""Statistics sampling failures and expiration never disturb catalog reconciliation.""" import json import os +import sqlite3 +import subprocess import sys import time +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any +from uuid import uuid4 import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + StatisticsHorizonRejectionTestCase, + StatisticsHorizonTestCase, StatisticsRecoveryTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( @@ -19,7 +25,9 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( location_process, open_legacy_warehouse, + run_statistics_retention, snapshot_api, + statistics_relations, wait_for_generation, ) @@ -99,3 +107,100 @@ def test_given_relation_dropped_during_sampling_when_read_fails_then_catalog_can == test_case.expected_catalog_presence ) assert "Warehouse history read failed" in log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + StatisticsHorizonTestCase( + "Default horizon keeps a two-day-old sample", + (), + ("customers_two_days", "customers_recent"), + ), + StatisticsHorizonTestCase( + "One-day horizon deletes the two-day-old sample", + ("--statistics-retention-days", "1"), + ("customers_recent",), + ), + ], + ids=lambda case: case.description, +) +def test_given_configured_statistics_horizon_when_server_starts_then_older_samples_are_deleted( + test_case: StatisticsHorizonTestCase, tmp_path: Path +) -> None: + record: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json").read_text() + )["record"] + _ = run_statistics_retention(tmp_path, ()) + now: datetime = datetime.now(UTC) + location_id: str = str(uuid4()) + with sqlite3.connect(tmp_path / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_locations (location_id,name,created_by,created_at) " + "VALUES (?, 'orders', 'test-admin', ?)", + (location_id, now.isoformat()), + ) + _ = connection.executemany( + "INSERT INTO sqb_warehouse_statistics (event_id,location_id,target,state_database," + "state_schema,relation_name,observed_at,record_json,received_at) " + "VALUES (?,?,'development',NULL,'main',?,?,?,?)", + [ + ( + str(uuid4()), + location_id, + name, + (now - age).isoformat(), + json.dumps(record | {"name": name}), + now.isoformat(), + ) + for name, age in [ + ("sentinel_expired", timedelta(days=100)), + ("customers_two_days", timedelta(days=2)), + ("customers_recent", timedelta(hours=1)), + ] + ], + ) + log: str = run_statistics_retention(tmp_path, test_case.server_arguments) + assert "Control API ready" in log + assert statistics_relations(tmp_path) == test_case.expected_relations + + +@pytest.mark.parametrize( + "test_case", + [ + StatisticsHorizonRejectionTestCase( + "Zero days", "0", "statistics retention must keep at least one day" + ), + StatisticsHorizonRejectionTestCase( + "Beyond the supported range", + "65536", + "--statistics-retention-days must be between 1 and 65535", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_statistics_horizon_when_server_starts_then_it_refuses_before_storage( + test_case: StatisticsHorizonRejectionTestCase, tmp_path: Path +) -> None: + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "server", + "--port", + "0", + "--statistics-retention-days", + test_case.days, + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_error in result.stderr + assert result.stdout == "" + assert not (tmp_path / ".sqlbuild/control.sqlite3").exists() From ef69e50dc74e57e03e0ea46c3cb4d755f1c09224 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:20:31 +0000 Subject: [PATCH 124/138] docs: describe the statistics retention option for sqb server --- .../sqlbuild/references/docs/cli/server.md | 16 ++++++++++++++++ website/src/content/docs/docs/cli/server.mdx | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index c846e07522..a58fe032b1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -14,6 +14,22 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Set retention + +Set how long the server keeps hourly warehouse statistics: + +```bash +sqb server --statistics-retention-days 30 +``` + +| Data | Option | Default | Range | +| --- | --- | --- | --- | +| Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | + +Samples older than the horizon disappear from statistics reads at once and are then deleted in the +background. An invalid value stops the server before it opens the control database. Servers that +share one control database delete at the shortest horizon among them, so give them the same value. + ## Inspect published SQL Sign in with [`sqb login`](login.md), then inspect one exact node in the current diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index c8b333552c..075957ba6f 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -13,6 +13,22 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Set retention + +Set how long the server keeps hourly warehouse statistics: + +```bash +sqb server --statistics-retention-days 30 +``` + +| Data | Option | Default | Range | +| --- | --- | --- | --- | +| Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | + +Samples older than the horizon disappear from statistics reads at once and are then deleted in the +background. An invalid value stops the server before it opens the control database. Servers that +share one control database delete at the shortest horizon among them, so give them the same value. + ## Inspect published SQL Sign in with [`sqb login`](/docs/cli/login/), then inspect one exact node in the current From 3531a177d052e6301ec73094cbf488c8a1b8c227 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:48:19 +0000 Subject: [PATCH 125/138] fix: name the retention flag when zero days are rejected --- .../src/native_commands/_helpers/decision.rs | 13 ++++++------- .../compiled_project/publication/test_statistics.py | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index d67dca4e01..d49566dc67 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -265,14 +265,13 @@ pub(crate) fn control_address( pub(crate) fn control_api_settings( namespace: &Namespace, ) -> Result<(ApiAuthentication, StatisticsRetention), ControlError> { + let out_of_range = + || ControlError::Invalid("--statistics-retention-days must be between 1 and 65535".into()); let retention = match namespace.get("control_statistics_retention_days") { - Some(Value::Int(value)) => { - StatisticsRetention::days(value.parse::().map_err(|_| { - ControlError::Invalid( - "--statistics-retention-days must be between 1 and 65535".into(), - ) - })?)? - } + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| out_of_range()) + .and_then(|days| StatisticsRetention::days(days).map_err(|_| out_of_range()))?, _ => StatisticsRetention::default(), }; Ok((control_authentication(namespace)?, retention)) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py index ae788c10b8..d0cd9f54ea 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py @@ -169,7 +169,7 @@ def test_given_configured_statistics_horizon_when_server_starts_then_older_sampl "test_case", [ StatisticsHorizonRejectionTestCase( - "Zero days", "0", "statistics retention must keep at least one day" + "Zero days", "0", "--statistics-retention-days must be between 1 and 65535" ), StatisticsHorizonRejectionTestCase( "Beyond the supported range", From 45d441f756eaf9ab52951e392b8665a54ee942de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:40:47 +0000 Subject: [PATCH 126/138] feat: expire security audit records after a configurable horizon sqb server --audit-retention-days N (default 400) sets how long security audit records are kept. A record older than the horizon is deleted only when every configured audit webhook has received it, no retained run's retry lookup depends on it, and it is not the newest record, so storage IDs are never reused. Records kept past the horizon are reported on stderr as "retained past horizon: N" whenever the count changes. --- .../security_audit_retention.json | 12 ++ .../sqlbuild-cli/src/argv/command_table.json | 11 ++ .../src/native_commands/_helpers/decision.rs | 34 ++-- .../src/native_commands/_helpers/execution.rs | 23 ++- .../src/native_commands/main/_run_control.rs | 6 +- .../credentials/_helpers/audit_retention.rs | 168 +++++++++++++++++ .../src/credentials/_helpers/mod.rs | 1 + .../src/credentials/constants.rs | 3 + .../src/credentials/main/audit/mod.rs | 1 + .../main/audit/run_audit_retention.rs | 42 +++++ .../src/credentials/models.rs | 32 ++++ .../src/runs/_helpers/requests/manual_job.rs | 3 +- .../runs/_helpers/requests/manual_rerun.rs | 3 +- .../src/runs/constants.rs | 4 + .../tests/credentials/helpers.rs | 78 +++++++- .../tests/credentials/test_audit_retention.rs | 173 ++++++++++++++++++ .../tests/credentials/test_contracts.rs | 2 + .../tests/credentials/test_types.rs | 6 + .../sqlbuild/references/docs/cli/server.md | 32 +++- .../publication/_test_types.py | 7 + .../compiled_project/publication/helpers.py | 37 +++- .../publication/test_audit_retention.py | 125 +++++++++++++ website/src/content/docs/docs/cli/server.mdx | 25 ++- 23 files changed, 786 insertions(+), 42 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json create mode 100644 crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs create mode 100644 crates/sqlbuild-orchestration/src/credentials/main/audit/run_audit_retention.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json b/benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json new file mode 100644 index 0000000000..c34d0ef6cb --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json @@ -0,0 +1,12 @@ +{ + "undelivered_past_horizon": { + "server_arguments": ["--audit-retention-days", "1"], + "rows": [ + {"action": "orders.delivered", "age_days": 2}, + {"action": "orders.undelivered", "age_days": 2}, + {"action": "orders.newest", "age_days": 0} + ], + "delivered_actions": ["orders.delivered"], + "expected_actions": ["orders.undelivered", "orders.newest"] + } +} diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 6c11446234..97f18690fd 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1154,6 +1154,17 @@ "help": "days of hourly warehouse statistics to keep; older samples are hidden and then deleted (default: 90)", "group": 1 }, + { + "kind": "store", + "options": [ + "--audit-retention-days" + ], + "dest": "control_audit_retention_days", + "default": 400, + "type": "int", + "help": "days of security audit records to keep; older records are deleted once every configured audit webhook has received them and no retained run's retry needs them (default: 400)", + "group": 1 + }, { "kind": "store", "options": ["--trusted-proxy-config"], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index d49566dc67..a2af1a8a75 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use serde_json::json; use sqlbuild_orchestration::api::models::ApiAuthentication; -use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScope, PrincipalKind}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; @@ -264,17 +264,29 @@ pub(crate) fn control_address( /// API settings are validated together before any store is opened. pub(crate) fn control_api_settings( namespace: &Namespace, -) -> Result<(ApiAuthentication, StatisticsRetention), ControlError> { - let out_of_range = - || ControlError::Invalid("--statistics-retention-days must be between 1 and 65535".into()); - let retention = match namespace.get("control_statistics_retention_days") { - Some(Value::Int(value)) => value - .parse::() - .map_err(|_| out_of_range()) - .and_then(|days| StatisticsRetention::days(days).map_err(|_| out_of_range()))?, - _ => StatisticsRetention::default(), +) -> Result<(ApiAuthentication, (StatisticsRetention, AuditRetention)), ControlError> { + let statistics = match retention_days(namespace, "statistics")? { + Some(days) => StatisticsRetention::days(days)?, + None => StatisticsRetention::default(), + }; + let audit = match retention_days(namespace, "audit")? { + Some(days) => AuditRetention::days(days)?, + None => AuditRetention::default(), }; - Ok((control_authentication(namespace)?, retention)) + Ok((control_authentication(namespace)?, (statistics, audit))) +} + +/// Zero and out-of-range values name the flag before any type-specific check runs. +fn retention_days(namespace: &Namespace, kind: &str) -> Result, ControlError> { + match namespace.get(&format!("control_{kind}_retention_days")) { + Some(Value::Int(value)) => match value.parse::() { + Ok(days) if days > 0 => Ok(Some(days)), + _ => Err(ControlError::Invalid(format!( + "--{kind}-retention-days must be between 1 and 65535" + ))), + }, + _ => Ok(None), + } } fn control_authentication(namespace: &Namespace) -> Result { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 38deaddb3e..043b0b9682 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -13,7 +13,8 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, DockerImageSource}; use sqlbuild_orchestration::authorization::main::recover_administrator::recover_administrator; -use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::credentials::main::audit::run_audit_retention::run_audit_retention; +use sqlbuild_orchestration::credentials::models::{AuditContext, AuditRetention}; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::rebuild_projections::rebuild_projections; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; @@ -120,7 +121,7 @@ pub(crate) async fn run_development_roles( ))])); }, development_stopped(shutdown.clone())), )); - start_platform_roles(&mut tasks, (&store, &shutdown))?; + start_platform_roles(&mut tasks, (&store, &shutdown), AuditRetention::default())?; tasks.spawn(development_location_role( (&project, &executable), (location_id, &target), @@ -821,6 +822,7 @@ fn location_child(options: &LocationServe, script: &Path) -> Command { pub(crate) fn start_platform_roles( tasks: &mut JoinSet>, (store, shutdown): (&ControlStore, &watch::Receiver), + audit: AuditRetention, ) -> Result<(), ControlError> { let mut sinks: Vec<(Webhook, &str)> = Vec::new(); for (variable, source, label) in [ @@ -858,6 +860,23 @@ pub(crate) fn start_platform_roles( Stream::Stderr, "Platform watchdog started\n", )])); + let audit_store = store.clone(); + let audit_shutdown = shutdown.clone(); + let audit_sinks: Vec = sinks.iter().map(|(sink, _)| sink.clone()).collect(); + tasks.spawn(async move { + let report = |retained: i64| { + drop(write_output(&[( + Stream::Stderr, + &format!("Security audit retention: retained past horizon: {retained}\n"), + )])); + }; + run_audit_retention( + &audit_store, + (audit, &audit_sinks), + (report, audit_shutdown), + ) + .await + }); for (sink, label) in sinks { drop(write_output(&[( Stream::Stderr, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 2656a38d48..2fcc53cce7 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -27,7 +27,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let (authentication, retention) = control_api_settings(namespace)?; + let (authentication, (statistics, audit)) = control_api_settings(namespace)?; let store = open_control_store(namespace).await?; let (workers, images) = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); @@ -43,7 +43,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - (authentication, images, retention), + (authentication, images, statistics), ( move || { drop(write_output(&[( @@ -54,7 +54,7 @@ pub(crate) async fn run_control( stopped(shutdown.clone()), ), )); - start_platform_roles(&mut tasks, (&store, &shutdown))?; + start_platform_roles(&mut tasks, (&store, &shutdown), audit)?; } if standalone || !flag(namespace, "control_no_scheduler") { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs new file mode 100644 index 0000000000..e0bf08fe31 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs @@ -0,0 +1,168 @@ +//! Audit expiry is bounded cleanup that never outruns a sink, a retry lookup or the storage order. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::credentials::constants::AUDIT_RETENTION_BATCH; +use crate::credentials::models::AuditRetention; +use crate::runs::constants::{MANUAL_JOB_AUDIT_TAG, RERUN_AUDIT_TAG}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// One full pass in storage order; returns rows deleted and rows still kept past the horizon. +pub(crate) async fn prune_audit( + store: &ControlStore, + retention: AuditRetention, + sinks: &[String], +) -> Result<(usize, i64), ControlError> { + let mut deleted = 0; + let mut after = 0; + loop { + let (count, next) = prune_audit_batch(store, (retention, sinks), after).await?; + deleted += count; + match next { + Some(next) => after = next, + None => break, + } + } + let transaction = begin(store, TransactionMode::Read).await?; + let cutoff = retention.cutoff(transaction.now); + let (transaction, rows) = query( + transaction, + "SELECT COUNT(*) AS retained FROM sqb_security_audit WHERE occurred_at<$1", + vec![Parameter::Time(cutoff)], + ) + .await?; + commit(transaction).await?; + let retained = rows + .first() + .and_then(|row| row["retained"].as_i64()) + .unwrap_or_default(); + Ok((deleted, retained)) +} + +/// The newest row is never deleted, so the next MAX+1 storage ID is never reused. +async fn prune_audit_batch( + store: &ControlStore, + (retention, sinks): (AuditRetention, &[String]), + after: i64, +) -> Result<(usize, Option), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let cutoff: DateTime = retention.cutoff(transaction.now); + let acknowledged: String = (0..sinks.len()) + .map(|index| { + format!( + " AND EXISTS (SELECT 1 FROM sqb_security_audit_deliveries d \ + WHERE d.sink_id=${} AND d.storage_id=a.storage_id)", + index + 4 + ) + }) + .collect(); + let mut parameters = vec![ + Parameter::Time(cutoff), + Parameter::Integer(after), + Parameter::Integer(AUDIT_RETENTION_BATCH as i64), + ]; + parameters.extend(sinks.iter().cloned().map(Parameter::Text)); + let (transaction, rows) = query( + transaction, + &format!( + "SELECT a.storage_id, a.event_id, a.scope_json FROM sqb_security_audit a \ + WHERE a.occurred_at<$1 AND a.storage_id>$2 \ + AND a.storage_id<(SELECT MAX(storage_id) FROM sqb_security_audit){acknowledged} \ + ORDER BY a.storage_id LIMIT $3" + ), + parameters, + ) + .await?; + let next = (rows.len() == AUDIT_RETENTION_BATCH) + .then(|| rows.last().and_then(|row| row["storage_id"].as_i64())) + .flatten(); + let (transaction, referenced) = retry_referenced(transaction, &rows).await?; + let expired: Vec = rows + .iter() + .filter_map(|row| row["storage_id"].as_i64()) + .filter(|storage_id| !referenced.contains(storage_id)) + .map(Parameter::Integer) + .collect(); + let mut transaction = transaction; + if !expired.is_empty() { + let placeholders = (1..=expired.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + for table in ["sqb_security_audit_deliveries", "sqb_security_audit"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE storage_id IN ({placeholders})"), + expired.clone(), + ) + .await?; + } + } + commit(transaction).await?; + Ok((expired.len(), next)) +} + +/// Mirrors the retry join; a row whose identifiers cannot be read is kept, not guessed about. +async fn retry_referenced( + transaction: ControlTransaction, + rows: &[Row], +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let mut kept: BTreeSet = BTreeSet::new(); + let mut candidates: Vec<(i64, Uuid)> = Vec::new(); + for row in rows { + let Some(storage_id) = row["storage_id"].as_i64() else { + continue; + }; + let Some(run_id) = row["scope_json"].get("run_id") else { + continue; + }; + let (Ok(event_id), Ok(run_id)) = ( + serde_json::from_value::(row["event_id"].clone()), + serde_json::from_value::(run_id.clone()), + ) else { + kept.insert(storage_id); + continue; + }; + if Uuid::new_v5(&run_id, MANUAL_JOB_AUDIT_TAG) == event_id + || Uuid::new_v5(&run_id, RERUN_AUDIT_TAG) == event_id + { + candidates.push((storage_id, run_id)); + } + } + if candidates.is_empty() { + return Ok((transaction, kept)); + } + let placeholders = (1..=candidates.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let (transaction, runs) = query( + transaction, + &format!("SELECT run_id FROM sqb_runs WHERE run_id IN ({placeholders})"), + candidates + .iter() + .map(|(_, run_id)| Parameter::Id(*run_id)) + .collect(), + ) + .await?; + let mut retained: BTreeSet = BTreeSet::new(); + for row in runs { + retained.insert( + serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + } + kept.extend( + candidates + .into_iter() + .filter(|(_, run_id)| retained.contains(run_id)) + .map(|(storage_id, _)| storage_id), + ); + Ok((transaction, kept)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs index 723c58f4d1..020e67bc15 100644 --- a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod active_session; +pub(crate) mod audit_retention; pub(crate) mod browser_sessions; pub(crate) mod locations; pub(crate) mod password_sessions; diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs index 26dc2378bc..a47221ed7d 100644 --- a/crates/sqlbuild-orchestration/src/credentials/constants.rs +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -15,3 +15,6 @@ pub(crate) const LOGIN_PEER_LIMIT: u64 = 20; pub(crate) const LOGIN_USERNAME_BYTES: usize = 256; /// A noncredential PHC output makes missing-user checks perform the same Argon2id work. pub(crate) const DUMMY_PASSWORD_HASH: &str = "$argon2id$v=19$m=19456,t=2,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + +pub(crate) const DEFAULT_AUDIT_RETENTION_DAYS: u16 = 400; +pub(crate) const AUDIT_RETENTION_BATCH: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs index ef2b9bf9b0..976ebad1c3 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod append_audit; pub mod record_authentication_denial; pub mod reserve_authentication_attempt; +pub mod run_audit_retention; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/run_audit_retention.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/run_audit_retention.rs new file mode 100644 index 0000000000..2e674bed6a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/run_audit_retention.rs @@ -0,0 +1,42 @@ +//! Expire audit rows periodically and say how many outlive the horizon, and why that may grow. + +use std::time::Duration; + +use tokio::sync::watch; + +use crate::credentials::_helpers::audit_retention::prune_audit; +use crate::credentials::models::AuditRetention; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::webhooks::models::{Webhook, WebhookSource}; + +/// Only this server's security-audit sinks acknowledge; `report` hears each new retained count. +pub async fn run_audit_retention( + store: &ControlStore, + (retention, sinks): (AuditRetention, &[Webhook]), + (report, mut shutdown): (impl Fn(i64), watch::Receiver), +) -> Result<(), ControlError> { + let sinks: Vec = sinks + .iter() + .filter(|sink| matches!(sink.source, WebhookSource::SecurityAudit)) + .map(|sink| sink.sink_id.clone()) + .collect(); + let mut reported = 0; + while !*shutdown.borrow() { + match prune_audit(store, retention, &sinks).await { + Ok((_, retained)) => { + if retained != reported { + report(retained); + } + reported = retained; + } + Err(ControlError::Database(error)) if ControlError::transient_database(&error) => {} + Err(error) => return Err(error), + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(60)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index 1458654cbf..2cdbc21af2 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -6,6 +6,9 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::credentials::constants::DEFAULT_AUDIT_RETENTION_DAYS; +use crate::store::errors::ControlError; + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum PrincipalKind { @@ -54,6 +57,35 @@ pub struct SecurityAuditEvent { pub after_json: serde_json::Value, } +/// Installation configuration: rows past the horizon stay until every sink and retry lookup is done. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AuditRetention { + days: u16, +} + +impl AuditRetention { + pub fn days(days: u16) -> Result { + if days == 0 { + return Err(ControlError::Invalid( + "audit retention must keep at least one day".into(), + )); + } + Ok(Self { days }) + } + + pub(crate) fn cutoff(self, now: DateTime) -> DateTime { + now - chrono::Duration::days(i64::from(self.days)) + } +} + +impl Default for AuditRetention { + fn default() -> Self { + Self { + days: DEFAULT_AUDIT_RETENTION_DAYS, + } + } +} + /// Closed permission vocabulary; unknown actions fail deserialization. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub enum Action { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index 1477cd8a15..1cc6539438 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -8,6 +8,7 @@ use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::constants::MANUAL_JOB_AUDIT_TAG; use crate::runs::main::frozen_job_request::frozen_job_request; use crate::runs::models::{JobRunRequest, RunWindow}; use crate::store::errors::ControlError; @@ -52,7 +53,7 @@ pub(crate) async fn request_job_run_in( "backfill window start must not follow end".into(), )); } - let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); + let event_id = Uuid::new_v5(&request.run_id, MANUAL_JOB_AUDIT_TAG); let (transaction, retry) = request_retry( transaction, &context, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs index d9e48554c8..df6dc2e6ce 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs @@ -8,6 +8,7 @@ use crate::credentials::models::{Action, AuditContext}; use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::constants::RERUN_AUDIT_TAG; use crate::runs::models::{RerunRequest, RunRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -53,7 +54,7 @@ pub(crate) async fn request_rerun_in( "rerun needs an actor, granting role and reason".into(), )); } - let event_id = Uuid::new_v5(&request.run_id, b"rerun-request"); + let event_id = Uuid::new_v5(&request.run_id, RERUN_AUDIT_TAG); let (transaction, retry) = request_retry( transaction, &context, diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 661f08ec4f..4d273c9a15 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -10,3 +10,7 @@ pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; pub(crate) const RECALL_BATCH_ROWS: usize = 500; pub(crate) const RECALL_INVOCATION_ID_BYTES: usize = 256; + +/// Manual requests derive their audit event ID from the run ID so retries find the original. +pub(crate) const MANUAL_JOB_AUDIT_TAG: &[u8] = b"manual-job-request"; +pub(crate) const RERUN_AUDIT_TAG: &[u8] = b"rerun-request"; diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs index cf1f753f11..4a17d5210b 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -1,4 +1,7 @@ -use crate::store::models::ControlStore; +use uuid::Uuid; + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ @@ -41,9 +44,6 @@ pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { pub(crate) async fn authentication_revision( store: &ControlStore, ) -> Result { - use crate::store::main::{begin::begin, commit::commit, query::query}; - use crate::store::models::TransactionMode; - let (transaction, rows) = query( begin(store, TransactionMode::Read).await?, "SELECT revision FROM sqb_authentication_revision WHERE singleton=1", @@ -58,3 +58,73 @@ pub(crate) async fn authentication_revision( crate::store::errors::ControlError::Schema("authentication revision is invalid".into()) }) } + +/// Test-only clock skew: every audit row becomes older than the given number of days. +pub(crate) async fn age_audit(store: &ControlStore, days: i64) { + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age audit"); + let then = transaction.now - chrono::Duration::days(days); + let (transaction, _) = query( + transaction, + "UPDATE sqb_security_audit SET occurred_at=$1", + vec![Parameter::Time(then)], + ) + .await + .expect("aged audit rows"); + commit(transaction).await.expect("commit aged audit"); +} + +/// Acknowledge every audit row up to and including `through` for one sink, in storage order. +pub(crate) async fn acknowledge_audit(store: &ControlStore, sink_id: &str, through: i64) { + let transaction = begin(store, TransactionMode::Write) + .await + .expect("acknowledge audit"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_security_audit_deliveries (sink_id, storage_id, event_id, delivered_at) \ + SELECT $1, storage_id, event_id, $2 FROM sqb_security_audit WHERE storage_id<=$3", + vec![ + Parameter::Text(sink_id.into()), + Parameter::Time(now), + Parameter::Integer(through), + ], + ) + .await + .expect("delivery rows"); + commit(transaction).await.expect("commit deliveries"); +} + +/// Remaining audit event IDs with their storage IDs, and the delivery rows that remain. +pub(crate) async fn audit_rows(store: &ControlStore) -> (Vec<(Uuid, i64)>, i64) { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read audit"); + let (transaction, rows) = query( + transaction, + "SELECT event_id, storage_id FROM sqb_security_audit ORDER BY storage_id", + vec![], + ) + .await + .expect("audit rows"); + let (transaction, deliveries) = query( + transaction, + "SELECT COUNT(*) AS deliveries FROM sqb_security_audit_deliveries", + vec![], + ) + .await + .expect("delivery count"); + commit(transaction).await.expect("commit audit read"); + ( + rows.iter() + .map(|row| { + ( + serde_json::from_value(row["event_id"].clone()).expect("event ID"), + row["storage_id"].as_i64().expect("storage ID"), + ) + }) + .collect(), + deliveries[0]["deliveries"].as_i64().expect("deliveries"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs new file mode 100644 index 0000000000..c799cb7108 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs @@ -0,0 +1,173 @@ +//! Audit expiry deletes only rows every sink received and no retained run's retry still reads. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::audit_retention::prune_audit; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, AuditRetention}; +use crate::credentials::tests::helpers::{acknowledge_audit, age_audit, audit_rows}; +use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::test_types::AuditRetentionTestCase; +use crate::runs::constants::MANUAL_JOB_AUDIT_TAG; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_expired_audit_rows_when_pruned_then_undelivered_and_retry_referenced_rows_survive() { + let test_cases = [ + AuditRetentionTestCase { + description: "SQLite keeps undelivered, retry-referenced and newest rows", + backend: Backend::Sqlite, + expected_retained: 3, + }, + AuditRetentionTestCase { + description: "PostgreSQL keeps undelivered, retry-referenced and newest rows", + backend: Backend::Postgres, + expected_retained: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "orders-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(store, "orders", "orders-admin") + .await + .expect("location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"daily_orders": { + "description": "Build orders", "selection": ["orders"], "settings": {}, + "concurrency_groups": [], "overlap": "queue" + }}), + ); + publish_snapshot(store, &input).await.expect("publish job"); + let delivered = Uuid::now_v7(); + let missing_run = Uuid::now_v7(); + let orphaned = Uuid::new_v5(&missing_run, MANUAL_JOB_AUDIT_TAG); + for (event_id, scope) in [ + (delivered, json!({"location_id": location_id})), + (orphaned, json!({"run_id": missing_run})), + ] { + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let transaction = append_audit( + transaction, + &context, + (event_id, "audit.retention_fixture"), + (scope, json!(null), json!({"recorded": true})), + ) + .await + .expect("append"); + commit(transaction).await.expect("commit"); + } + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh orders".into(), + priority: 1, + window: None, + }; + assert!( + request_job_run(store, &context, &request) + .await + .expect("request") + ); + let referenced = Uuid::new_v5(&request.run_id, MANUAL_JOB_AUDIT_TAG); + let (rows, _) = audit_rows(store).await; + let through = rows.last().expect("referenced row").1; + let undelivered = Uuid::now_v7(); + let newest = Uuid::now_v7(); + for event_id in [undelivered, newest] { + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let transaction = append_audit( + transaction, + &context, + (event_id, "audit.retention_fixture"), + (json!({}), json!(null), json!({"recorded": true})), + ) + .await + .expect("append"); + commit(transaction).await.expect("commit"); + } + age_audit(store, 401).await; + acknowledge_audit(store, "orders-audit-sink", through).await; + let sinks = ["orders-audit-sink".to_string()]; + assert!(AuditRetention::days(0).is_err()); + assert_eq!( + prune_audit(store, AuditRetention::days(500).expect("horizon"), &sinks) + .await + .expect("longer horizon") + .0, + 0 + ); + let two_sinks = [sinks[0].clone(), "inventory-audit-sink".to_string()]; + assert_eq!( + prune_audit(store, AuditRetention::default(), &two_sinks) + .await + .expect("second sink has received nothing") + .0, + 0 + ); + let total = rows.len() + 2; + let (deleted, retained) = prune_audit(store, AuditRetention::default(), &sinks) + .await + .expect("prune"); + assert_eq!( + retained, test_case.expected_retained, + "{}", + test_case.description + ); + assert_eq!(deleted, total - 3); + let (remaining, deliveries) = audit_rows(store).await; + let ids: Vec = remaining.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, [referenced, undelivered, newest]); + assert_eq!(deliveries, 1, "only the retained row's delivery remains"); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry still finds its original request") + ); + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let transaction = append_audit( + transaction, + &context, + (Uuid::now_v7(), "audit.retention_fixture"), + (json!({}), json!(null), json!({"recorded": true})), + ) + .await + .expect("append after prune"); + commit(transaction).await.expect("commit"); + let (after, _) = audit_rows(store).await; + assert_eq!( + after.last().expect("new row").1, + remaining.last().expect("newest").1 + 1, + "storage IDs are never reused" + ); + assert_eq!( + prune_audit(store, AuditRetention::default(), &sinks) + .await + .expect("idempotent"), + (0, test_case.expected_retained) + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index 0cfe559ea6..bcbd4651cd 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -8,6 +8,8 @@ pub(crate) mod location_helpers; mod test_administrators; #[path = "test_audit.rs"] mod test_audit; +#[path = "test_audit_retention.rs"] +mod test_audit_retention; #[path = "test_contract.rs"] mod test_contract; #[path = "passwords/test_contract.rs"] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index a2bf39bdbb..00c52cf199 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -34,3 +34,9 @@ pub(crate) struct AuditWebhookReceiver { pub(crate) requests: Arc>>, pub(crate) status: Arc, } + +pub(crate) struct AuditRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_retained: i64, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index a58fe032b1..548327cacf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/cli/server/ +## Contents + +- Set retention +- Inspect published SQL +- Repair control views +- Handle a refused repair + Run the control API and scheduler without loading project Python: ```bash @@ -16,19 +23,34 @@ Use `sqb server --help` for service, authentication and launcher options. ## Set retention -Set how long the server keeps hourly warehouse statistics: +Set how long the server keeps hourly warehouse statistics and security audit records: ```bash -sqb server --statistics-retention-days 30 +sqb server --statistics-retention-days 30 --audit-retention-days 730 ``` | Data | Option | Default | Range | | --- | --- | --- | --- | | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | +| Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | + +Statistics samples older than the horizon disappear from statistics reads at once and are then +deleted in the background. + +A security audit record older than the horizon is deleted only when it is no longer needed: + +- The webhook in `SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL`, if set, has received it. +- No retained run's retry depends on it. A manual job or rerun request keeps its record while + the run exists. +- It is not the newest record, so audit page cursors never see a reused position. + +Records kept past the horizon for these reasons are reported on stderr as +`Security audit retention: retained past horizon: N` whenever the count changes. A growing count +usually means the audit webhook is failing. -Samples older than the horizon disappear from statistics reads at once and are then deleted in the -background. An invalid value stops the server before it opens the control database. Servers that -share one control database delete at the shortest horizon among them, so give them the same value. +An invalid value stops the server before it opens the control database. Servers that share one +control database delete at the shortest horizon among them and count only their own audit +webhook, so give them the same values and webhook. ## Inspect published SQL diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 005457e7e2..d3b3efbc9e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -265,6 +265,13 @@ class StatisticsHorizonTestCase: expected_relations: tuple[str, ...] +@dataclass(frozen=True) +class AuditHorizonTestCase: + description: str + fixture: str + expected_retained: int + + @dataclass(frozen=True) class StatisticsHorizonRejectionTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a53fb60d89..286d02156c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1010,19 +1010,27 @@ def read_columns(**scope: Any) -> Any: def run_statistics_retention(directory: Path, arguments: tuple[str, ...]) -> str: """Run one installed server until its first retention pass, returning its stderr.""" + return run_server_until( + directory, + ("--no-scheduler", *arguments), + lambda _: "sentinel_expired" not in statistics_relations(directory), + ) + + +def run_server_until( + directory: Path, + arguments: tuple[str, ...], + finished: Callable[[str], bool], + environment_overrides: dict[str, str] | None = None, +) -> str: + """Run one installed server until its stderr or store satisfies `finished`, returning stderr.""" log: Path = directory / "server.log" environment: dict[str, str] = os.environ.copy() environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + environment.update(environment_overrides or {}) with log.open("wb") as stderr: process: subprocess.Popen[bytes] = subprocess.Popen( - [ - str(Path(sys.executable).with_name("sqb")), - "server", - "--port", - "0", - "--no-scheduler", - *arguments, - ], + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", *arguments], cwd=directory, env=environment, stdout=subprocess.DEVNULL, @@ -1031,9 +1039,9 @@ def run_statistics_retention(directory: Path, arguments: tuple[str, ...]) -> str try: _ = api_address(process, log) deadline: float = time.monotonic() + 10 - while "sentinel_expired" in statistics_relations(directory): + while not finished(log.read_text()): assert process.poll() is None, log.read_text() - assert time.monotonic() < deadline, "retention did not run at startup" + assert time.monotonic() < deadline, "server did not reach the expected state" time.sleep(0.05) finally: process.terminate() @@ -1041,6 +1049,15 @@ def run_statistics_retention(directory: Path, arguments: tuple[str, ...]) -> str return log.read_text() +def audit_actions(directory: Path) -> tuple[str, ...]: + """Read the synthetic retention fixture's audit actions directly, in storage order.""" + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT action FROM sqb_security_audit WHERE action LIKE 'orders.%' ORDER BY storage_id" + ).fetchall() + return tuple(action for (action,) in rows) + + def statistics_relations(directory: Path) -> tuple[str, ...]: """Read the retained sample relations directly, oldest first.""" with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py new file mode 100644 index 0000000000..b0ec7bffc4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py @@ -0,0 +1,125 @@ +"""The installed server expires security audit rows only after every configured sink has them.""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + AuditHorizonTestCase, + StatisticsHorizonRejectionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + audit_actions, + run_server_until, +) + +UNREACHABLE_SINK: str = "http://127.0.0.1:9/orders-audit" + + +@pytest.mark.parametrize( + "test_case", + [ + AuditHorizonTestCase( + "Configured sink keeps the row it has not received", "undelivered_past_horizon", 1 + ) + ], + ids=lambda case: case.description, +) +def test_given_expired_audit_rows_when_server_runs_then_only_delivered_rows_are_deleted( + test_case: AuditHorizonTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json" + ).read_text() + )[test_case.fixture] + _ = run_server_until(tmp_path, ("--no-scheduler",), lambda log: "Control API ready" in log) + now: datetime = datetime.now(UTC) + sink_id: str = hashlib.sha256(UNREACHABLE_SINK.encode()).hexdigest() + with sqlite3.connect(tmp_path / ".sqlbuild/control.sqlite3") as connection: + (last,) = connection.execute( + "SELECT COALESCE(MAX(storage_id), 0) FROM sqb_security_audit" + ).fetchone() + _ = connection.executemany( + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, " + "request_id, ip_address, scope_json, before_json, after_json, storage_id) " + "VALUES (?, ?, 'orders-admin', ?, 'admin', ?, NULL, '{}', 'null', '{}', ?)", + [ + ( + str(uuid4()), + (now - timedelta(days=row["age_days"])).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + row["action"], + str(uuid4()), + last + offset, + ) + for offset, row in enumerate(fixture["rows"], start=1) + ], + ) + _ = connection.execute( + "INSERT INTO sqb_security_audit_deliveries (sink_id, storage_id, event_id, delivered_at) " + "SELECT ?, storage_id, event_id, occurred_at FROM sqb_security_audit " + "WHERE action IN (SELECT value FROM json_each(?))", + (sink_id, json.dumps(fixture["delivered_actions"])), + ) + log: str = run_server_until( + tmp_path, + ("--no-scheduler", *fixture["server_arguments"]), + lambda _: audit_actions(tmp_path) == tuple(fixture["expected_actions"]), + {"SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL": UNREACHABLE_SINK}, + ) + assert f"Security audit retention: retained past horizon: {test_case.expected_retained}" in log + with sqlite3.connect(tmp_path / ".sqlbuild/control.sqlite3") as connection: + deliveries: list[tuple[int]] = connection.execute( + "SELECT storage_id FROM sqb_security_audit_deliveries" + ).fetchall() + assert deliveries == [] + + +@pytest.mark.parametrize( + "test_case", + [ + StatisticsHorizonRejectionTestCase( + "Zero days", "0", "--audit-retention-days must be between 1 and 65535" + ), + StatisticsHorizonRejectionTestCase( + "Beyond the supported range", + "65536", + "--audit-retention-days must be between 1 and 65535", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_audit_horizon_when_server_starts_then_it_refuses_before_storage( + test_case: StatisticsHorizonRejectionTestCase, tmp_path: Path +) -> None: + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "server", + "--port", + "0", + "--audit-retention-days", + test_case.days, + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_error in result.stderr + assert result.stdout == "" + assert not (tmp_path / ".sqlbuild/control.sqlite3").exists() diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 075957ba6f..b46a7f7233 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -15,19 +15,34 @@ Use `sqb server --help` for service, authentication and launcher options. ## Set retention -Set how long the server keeps hourly warehouse statistics: +Set how long the server keeps hourly warehouse statistics and security audit records: ```bash -sqb server --statistics-retention-days 30 +sqb server --statistics-retention-days 30 --audit-retention-days 730 ``` | Data | Option | Default | Range | | --- | --- | --- | --- | | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | +| Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | -Samples older than the horizon disappear from statistics reads at once and are then deleted in the -background. An invalid value stops the server before it opens the control database. Servers that -share one control database delete at the shortest horizon among them, so give them the same value. +Statistics samples older than the horizon disappear from statistics reads at once and are then +deleted in the background. + +A security audit record older than the horizon is deleted only when it is no longer needed: + +- The webhook in `SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL`, if set, has received it. +- No retained run's retry depends on it. A manual job or rerun request keeps its record while + the run exists. +- It is not the newest record, so audit page cursors never see a reused position. + +Records kept past the horizon for these reasons are reported on stderr as +`Security audit retention: retained past horizon: N` whenever the count changes. A growing count +usually means the audit webhook is failing. + +An invalid value stops the server before it opens the control database. Servers that share one +control database delete at the shortest horizon among them and count only their own audit +webhook, so give them the same values and webhook. ## Inspect published SQL From 0220cb2131e892d9a861a327117e069c2ff14b4e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:01:47 +0000 Subject: [PATCH 127/138] feat: expire old skipped and covered schedule ticks The scheduler leader deletes skipped and covered ticks older than 30 days in fenced batches, keeping each trigger's latest tick and every tick a run points at. Evaluation never starts before the same horizon, so a new schedule, a resumed trigger or a cursor reset catches up from 30 days ago at most and deleted ticks are never recreated. Additive indexes keep the cleanup bounded. --- .../orchestration/tick_retention.json | 6 + .../scheduler/_helpers/drain_due_triggers.rs | 25 +++ .../src/scheduler/_helpers/mod.rs | 1 + .../src/scheduler/_helpers/tick_retention.rs | 82 +++++++++ .../src/scheduler/constants.rs | 4 + .../src/scheduler/main/_evaluate_trigger.rs | 9 +- .../src/scheduler/main/_scheduler_cycle.rs | 27 +-- .../src/scheduler/models.rs | 2 + .../src/store/_helpers/migration.rs | 2 + .../src/store/retention_indexes.sql | 2 + .../tests/api/test_changes.rs | 5 +- .../tests/api/test_external_sessions.rs | 1 + .../tests/api/test_reads.rs | 7 +- .../tests/api/warehouse_metadata/helpers.rs | 2 +- .../tests/credentials/test_audit.rs | 1 + .../tests/platform/helpers.rs | 11 +- .../tests/runs/test_coalesce.rs | 13 +- .../tests/scheduler/ticks/helpers.rs | 131 ++++++++++++++- .../tests/scheduler/ticks/test_progress.rs | 13 +- .../tests/scheduler/ticks/test_retention.rs | 155 ++++++++++++++++++ .../tests/scheduler/ticks/test_ticks.rs | 2 + .../tests/scheduler/ticks/test_triggers.rs | 24 +-- .../tests/scheduler/ticks/test_types.rs | 6 + .../tests/sensors/test_sources.rs | 1 + .../tests/snapshots/test_replay.rs | 5 +- .../tests/store/helpers.rs | 1 + .../sqlbuild/references/docs/cli/server.md | 6 + website/src/content/docs/docs/cli/server.mdx | 6 + 28 files changed, 474 insertions(+), 76 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/tick_retention.json create mode 100644 crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs create mode 100644 crates/sqlbuild-orchestration/src/store/retention_indexes.sql create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json new file mode 100644 index 0000000000..b5f01f704b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json @@ -0,0 +1,6 @@ +{ + "description": "Covered ticks age past the tick horizon and are pruned; the requested tick and the latest covered tick stay; the operator then resets the cursor.", + "shift_days": 40, + "expected_pruned": 2, + "expected_survivors": ["covered", "requested"] +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs index ba0d626c09..7161888c49 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -1,5 +1,8 @@ //! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. +use std::time::Duration; + +use chrono::{DateTime, Utc}; use uuid::Uuid; use crate::scheduler::_helpers::drain_sensors::drain_sensors; @@ -38,3 +41,25 @@ pub(crate) async fn drain_due_triggers( cycle.evaluated += sensor_evaluations; Ok((lease, cycle)) } + +/// Shorten the cycle wait to the next running trigger's due time, never below zero. +pub(crate) async fn wait_for_next_due( + store: &ControlStore, + wait_for: Duration, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?; + let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + commit(transaction).await?; + Ok(next.map_or(wait_for, |next| { + (next - now) + .to_std() + .unwrap_or(Duration::ZERO) + .min(wait_for) + })) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs index c007a30d78..bee69ea1d3 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod drain_due_triggers; pub(crate) mod drain_sensors; pub(crate) mod expansion; pub(crate) mod persistence; +pub(crate) mod tick_retention; pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs new file mode 100644 index 0000000000..d3d5b21f08 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs @@ -0,0 +1,82 @@ +//! Tick expiry is fenced, batched cleanup that keeps each trigger's cursor tick and run links. + +use std::time::{Duration, Instant}; + +use chrono::{DateTime, TimeDelta, Utc}; +use uuid::Uuid; + +use crate::scheduler::constants::{ + TICK_RETENTION_BATCH, TICK_RETENTION_DAYS, TICK_RETENTION_INTERVAL_SECONDS, +}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +/// Evaluation never starts before the horizon, so a cursor reset cannot resurrect deleted ticks. +pub(crate) fn evaluation_start( + (last_tick_for, started_at): (Option>, DateTime), + now: DateTime, +) -> Result, ControlError> { + let start = match last_tick_for { + Some(last) => last, + None => started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + }; + Ok(start.max(now - TimeDelta::days(TICK_RETENTION_DAYS))) +} + +/// Run the leader's tick cleanup at most once per interval; returns when it last ran. +pub(crate) async fn prune_ticks_when_due( + store: &ControlStore, + lease: &SchedulerLease, + pruned_at: Option, +) -> Result, ControlError> { + let interval = Duration::from_secs(TICK_RETENTION_INTERVAL_SECONDS); + match pruned_at { + Some(at) if at.elapsed() < interval => Ok(Some(at)), + _ => { + prune_ticks(store, lease).await?; + Ok(Some(Instant::now())) + } + } +} + +/// Requested, failed and deferred ticks stay; so does any tick a run row points at. +pub(crate) async fn prune_ticks( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let mut deleted = 0; + loop { + let mut transaction = + check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let cutoff = transaction.now - TimeDelta::days(TICK_RETENTION_DAYS); + let rows; + (transaction, rows) = query( + transaction, + "DELETE FROM sqb_ticks WHERE tick_id IN (SELECT k.tick_id FROM sqb_ticks k \ + WHERE k.outcome IN ('skipped','covered') AND k.scheduled_for<$1 \ + AND k.scheduled_for<(SELECT MAX(l.scheduled_for) FROM sqb_ticks l WHERE l.trigger_id=k.trigger_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_runs r WHERE r.tick_id=k.tick_id) \ + ORDER BY k.scheduled_for, k.tick_id LIMIT $2) RETURNING trigger_id", + vec![ + Parameter::Time(cutoff), + Parameter::Integer(TICK_RETENTION_BATCH as i64), + ], + ) + .await?; + for row in &rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction.changes.insert(Change::Trigger(trigger_id)); + } + commit(check_lease(transaction, lease).await?).await?; + deleted += rows.len(); + if rows.len() < TICK_RETENTION_BATCH { + return Ok(deleted); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs index b6d1c0ff0f..b8c4f3b521 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/constants.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -6,3 +6,7 @@ pub const RENEW_INTERVAL_SECONDS: u64 = 5; pub const LEASE_TTL_SECONDS: i64 = 30; /// Bound each decision batch, including covered ticks, before opening a write transaction. pub const MAX_TICKS_PER_BATCH: usize = 10_000; +/// Skipped and covered ticks older than this are deleted, and no tick older than it is created. +pub(crate) const TICK_RETENTION_DAYS: i64 = 30; +pub(crate) const TICK_RETENTION_BATCH: usize = 100; +pub(crate) const TICK_RETENTION_INTERVAL_SECONDS: u64 = 60; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs index 2646e237ee..b6fe95739c 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -1,11 +1,11 @@ //! Fence, decide and publish using one serialized transaction and its database clock. -use chrono::TimeDelta; use uuid::Uuid; use crate::declarations::models::TriggerStatus; use crate::scheduler::_helpers::expansion::expand_schedule; use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::_helpers::tick_retention::evaluation_start; use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; @@ -42,12 +42,7 @@ pub(crate) async fn evaluate_trigger( return Ok(None); } let now = transaction.now; - let from = trigger.last_tick_for.unwrap_or( - trigger - .started_at - .checked_sub_signed(TimeDelta::microseconds(1)) - .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, - ); + let from = evaluation_start((trigger.last_tick_for, trigger.started_at), now)?; let (due_times, continuation) = expand_schedule(&trigger.definition_json, from, now)?; let (transaction, rows) = query(transaction, "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs index 1d94e6944a..c6267cfc84 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -2,20 +2,16 @@ use std::time::Duration; -use chrono::{DateTime, Utc}; - use crate::runs::constants::MONITOR_BATCH_SIZE; use crate::runs::main::monitor_runs::monitor_runs; -use crate::scheduler::_helpers::drain_due_triggers::drain_due_triggers; +use crate::scheduler::_helpers::drain_due_triggers::{drain_due_triggers, wait_for_next_due}; +use crate::scheduler::_helpers::tick_retention::prune_ticks_when_due; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::renew_lease::renew_lease; use crate::scheduler::models::{SchedulerCycle, SchedulerState}; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::ControlStore; pub(crate) async fn scheduler_cycle( store: &ControlStore, @@ -45,21 +41,8 @@ pub(crate) async fn scheduler_cycle( if cycle.unresponsive == MONITOR_BATCH_SIZE { cycle.wait_for = Duration::ZERO; } + state.pruned_at = prune_ticks_when_due(store, &lease, state.pruned_at).await?; state.lease = Some(lease); - let transaction = begin(store, TransactionMode::Read).await?; - let now = transaction.now; - let (transaction, rows) = query(transaction, - "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ - JOIN sqb_locations l ON l.location_id=t.location_id \ - WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?; - let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - if let Some(next) = next { - cycle.wait_for = (next - now) - .to_std() - .unwrap_or(Duration::ZERO) - .min(cycle.wait_for); - } - commit(transaction).await?; + cycle.wait_for = wait_for_next_due(store, cycle.wait_for).await?; Ok((state, cycle)) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index 39594ed147..c972d70b6c 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -19,6 +19,7 @@ pub struct SchedulerLease { pub struct SchedulerState { pub(crate) holder: Uuid, pub(crate) lease: Option, + pub(crate) pruned_at: Option, } impl SchedulerState { @@ -27,6 +28,7 @@ impl SchedulerState { Self { holder, lease: None, + pruned_at: None, } } } diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index d6a956106d..4575c78ee5 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -42,6 +42,7 @@ pub(crate) const WAREHOUSE_STATISTICS: &str = include_str!("../warehouse_statist pub(crate) const CHECK_OBSERVATIONS: &str = include_str!("../check_observations.sql"); pub(crate) const TRIGGER_PROGRESS: &str = include_str!("../trigger_progress.sql"); pub(crate) const CLI_LOGIN_FLOWS: &str = include_str!("../cli_login_flows.sql"); +pub(crate) const RETENTION_INDEXES: &str = include_str!("../retention_indexes.sql"); pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("location_instances", LOCATION_INSTANCES), @@ -76,6 +77,7 @@ pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("check_observations", CHECK_OBSERVATIONS), ("trigger_progress", TRIGGER_PROGRESS), ("cli_login_flows", CLI_LOGIN_FLOWS), + ("retention_indexes", RETENTION_INDEXES), ]; pub(crate) fn migration_digests(store: &ControlStore) -> Vec { diff --git a/crates/sqlbuild-orchestration/src/store/retention_indexes.sql b/crates/sqlbuild-orchestration/src/store/retention_indexes.sql new file mode 100644 index 0000000000..1cdd99cf66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/retention_indexes.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); +CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome IN ('skipped', 'covered'); diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs index 6b74d5a9d7..39c435ffb0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_changes.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -1,5 +1,6 @@ //! Real SQLite broadcast and PostgreSQL LISTEN/NOTIFY feed independent HTTP observers. +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use std::time::Duration; use chrono::Utc; @@ -72,8 +73,8 @@ fn given_independent_http_observers_when_committed_state_changes_then_scoped_sse .expect("location token"); let mut input = snapshot(location_id); input.documents.insert("jobs".into(), json!({"orders": {"description":"Build orders", "selection":["orders"], "overlap":"allow"}})); - input.documents.insert("triggers".into(), json!({"daily_orders": {"description":"Refresh orders", "job_key":"orders", "starts_at":"2020-01-01T00:00:00Z", - "schedule":{"kind":"calendar", "times":["2020-01-01T00:00:00Z","2030-01-01T00:00:00Z"], "refresh_at":"2029-01-01T00:00:00Z"}}})); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description":"Refresh orders", "job_key":"orders", "starts_at":calendar_text(0), + "schedule":{"kind":"calendar", "times":[calendar_text(0),"2030-01-01T00:00:00Z"], "refresh_at":"2029-01-01T00:00:00Z"}}})); publish_snapshot(store, &input) .await .expect("initial snapshot"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs index deeedc2e7a..055f5cd23f 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -94,6 +94,7 @@ fn given_federated_people_when_administrators_and_sessions_change_then_live_auth .expect("populated predecessor"); for sql in [ "DROP TABLE sqb_external_sessions", + "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='external_sessions'", ] { (transaction, _) = query(transaction, sql, vec![]) diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index 352df748dd..a9e7fe1903 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -1,5 +1,6 @@ //! Real HTTP reads keep both location and target scope, including guessed nested resource IDs. +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use std::collections::BTreeMap; use std::time::Duration; @@ -70,8 +71,8 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso "jobs".into(), json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), ); - input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", - "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}})); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}})); let published = publish_snapshot(store, &input) .await .expect("publish declarations"); @@ -337,7 +338,7 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso test_case.expected_ticks ); assert_eq!(ticks[0]["outcome"], "requested"); - assert_eq!(ticks[0]["scheduled_for"], "2020-01-03T00:00:00Z"); + assert_eq!(ticks[0]["scheduled_for"], calendar_text(2)); assert_eq!(ticks[1]["outcome"], "covered"); for path in [&run_path, &event_path, &tick_path, &invocation_path] { assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 4da2ffc4b0..7fbe5ab973 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -51,7 +51,7 @@ pub(crate) async fn fingerprint_history( .expect("predecessor table set"); let (transaction, _) = query( transaction, - "DELETE FROM sqb_schema_migrations WHERE name IN ('warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows','administrator_initialization','external_sessions')", + "DELETE FROM sqb_schema_migrations WHERE name IN ('retention_indexes','warehouse_fingerprints','development_locations','node_transitions','password_credentials','password_sessions','authentication_revision','oidc_login_flows','administrator_initialization','external_sessions')", vec![], ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 65f5fb076b..6734e12289 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -139,6 +139,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .expect("predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index e2da7ca896..842bcd81a4 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -1,3 +1,4 @@ +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; @@ -33,17 +34,17 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { format!("daily_orders_{index}"), json!({ "description": "Refresh orders", "job_key": "orders", - "starts_at": "2020-01-01T00:00:00Z", - "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", future], - "refresh_at": "2020-01-01T00:00:00Z"} + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), future], + "refresh_at": calendar_text(0)} }), ); } for (name, status) in [("expired_orders", "running"), ("paused_orders", "paused")] { triggers.insert(name.into(), json!({ "description": "Refresh orders", "job_key": "orders", "default_status": status, - "starts_at": "2020-01-01T00:00:00Z", - "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2020-01-01T00:00:00Z"} + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0)], "refresh_at": calendar_text(0)} })); } triggers.insert( diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index b9eb5090c4..ba16619fc2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -5,6 +5,7 @@ use crate::runs::main::claim_next_run::claim_next_run; use crate::runs::main::list_runs::list_runs; use crate::scheduler::models::TickDecision; use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use serde_json::json; @@ -46,8 +47,8 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks "jobs".into(), json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), ); - input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", - "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2030-01-01T00:00:00Z"}}})); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0)], "refresh_at": "2030-01-01T00:00:00Z"}}})); let published = publish_snapshot(store, &input) .await .expect("first calendar publication"); @@ -69,7 +70,7 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks assert_eq!(claim.request.snapshot_id, published.snapshot_id); input.generation = 2; input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = - json!(["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z"]); + json!([calendar_text(0), calendar_text(1)]); publish_snapshot(store, &input) .await .expect("refresh calendar"); @@ -84,11 +85,7 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks assert_ne!(waiting, claim.request.run_id); input.generation = 3; input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = - json!([ - "2020-01-01T00:00:00Z", - "2020-01-02T00:00:00Z", - "2020-01-03T00:00:00Z" - ]); + json!([calendar_text(0), calendar_text(1), calendar_text(2)]); publish_snapshot(store, &input) .await .expect("next calendar refresh"); diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 929015c698..705fee95e0 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -1,6 +1,9 @@ //! Trigger and failure fixtures exercise the scheduler's transactional tick boundary. use std::collections::BTreeMap; +use std::sync::LazyLock; + +use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Utc}; use crate::declarations::main::list_triggers::list_triggers; use crate::runs::main::list_runs::list_runs; @@ -12,7 +15,8 @@ use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; -use crate::store::models::ControlStore; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; use serde_json::{Value, json}; use uuid::Uuid; @@ -25,6 +29,20 @@ use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; use crate::store::tests::store_test_types::Backend; +/// Calendar fixtures stay inside the tick-retention horizon: day 0 is midnight three days ago. +pub(crate) fn calendar_time(day: i64) -> DateTime { + static BASE: LazyLock> = LazyLock::new(|| { + (Utc::now() - TimeDelta::days(3)) + .duration_trunc(TimeDelta::days(1)) + .expect("calendar base") + }); + *BASE + TimeDelta::days(day) +} + +pub(crate) fn calendar_text(day: i64) -> String { + calendar_time(day).to_rfc3339_opts(SecondsFormat::Secs, true) +} + pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { SnapshotInput { location_id, @@ -53,8 +71,8 @@ pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { ), ( "triggers".into(), - json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", - "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), ), ]), } @@ -288,3 +306,110 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi close_test_store(fixture).await; Some(counts) } + +/// Test-only clock skew: move a trigger's past calendar, cursor and ticks back by `days`. +pub(crate) async fn shift_schedule(store: &ControlStore, trigger_id: Uuid, days: i64) { + let shift = |value: &mut Value| { + let time: DateTime = serde_json::from_value(value.clone()).expect("calendar time"); + *value = json!(time - TimeDelta::days(days)); + }; + let past = |value: &&mut Value| { + serde_json::from_value::>((*value).clone()).expect("calendar time") + < Utc::now() + }; + let transaction = begin(store, TransactionMode::Write).await.expect("shift"); + let (transaction, rows) = query( + transaction, + "SELECT definition_json FROM sqb_triggers WHERE trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await + .expect("trigger definition"); + let mut definition = rows[0]["definition_json"].clone(); + shift(&mut definition["starts_at"]); + for time in definition["schedule"]["times"] + .as_array_mut() + .expect("calendar times") + .iter_mut() + .filter(past) + { + shift(time); + } + let (transaction, _) = query( + transaction, + "UPDATE sqb_triggers SET definition_json=$1 WHERE trigger_id=$2", + vec![Parameter::Json(definition), Parameter::Id(trigger_id)], + ) + .await + .expect("shift calendar"); + let (transaction, ticks) = query( + transaction, + "SELECT tick_id, scheduled_for FROM sqb_ticks WHERE trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await + .expect("ticks"); + let mut transaction = transaction; + for tick in ticks { + let mut time = tick["scheduled_for"].clone(); + shift(&mut time); + (transaction, _) = query( + transaction, + "UPDATE sqb_ticks SET scheduled_for=$1 WHERE tick_id=$2", + vec![ + Parameter::Time(serde_json::from_value(time).expect("tick time")), + Parameter::Id(serde_json::from_value(tick["tick_id"].clone()).expect("tick")), + ], + ) + .await + .expect("shift tick"); + } + let (transaction, state) = query( + transaction, + "SELECT started_at, last_tick_for FROM sqb_trigger_state WHERE trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await + .expect("trigger state"); + let started: DateTime = + serde_json::from_value(state[0]["started_at"].clone()).expect("start"); + let last: Option> = + serde_json::from_value(state[0]["last_tick_for"].clone()).expect("cursor"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=$2 WHERE trigger_id=$3", + vec![ + Parameter::Time(started - TimeDelta::days(days)), + Parameter::OptionalTime(last.map(|time| time - TimeDelta::days(days))), + Parameter::Id(trigger_id), + ], + ) + .await + .expect("shift state"); + commit(transaction).await.expect("commit shift"); +} + +/// A later covered tick, as if another due time arrived while the requested run still waited. +pub(crate) async fn append_covered_tick( + store: &ControlStore, + trigger_id: Uuid, + scheduled_for: DateTime, +) { + let transaction = begin(store, TransactionMode::Write) + .await + .expect("covered tick"); + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \ + run_id, covered_by_run_id, details_json) SELECT $1, trigger_id, $2, evaluated_at, epoch, \ + 'covered', NULL, run_id, details_json FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'", + vec![ + Parameter::Id(Uuid::now_v7()), + Parameter::Time(scheduled_for), + Parameter::Id(trigger_id), + ], + ) + .await + .expect("append covered tick"); + commit(transaction).await.expect("commit covered tick"); +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs index 522d574335..a1a9e3056a 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs @@ -10,7 +10,7 @@ use crate::declarations::main::{change_trigger::change_trigger, list_triggers::l use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; use crate::platform::main::rebuild_projections::rebuild_projections; use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; -use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::scheduler::tests::contracts::ticks::helpers::{calendar_text, calendar_time, snapshot}; use crate::scheduler::tests::contracts::ticks::test_types::TriggerControlsTestCase; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::{ @@ -83,10 +83,7 @@ fn given_reset_existing_ticks_when_progress_is_replayed_then_cursor_and_due_time .expect("reset state") .remove(0); assert!(reset.last_tick_for.is_none()); - assert_eq!( - reset.next_due_at.expect("first due").to_rfc3339(), - "2020-01-01T00:00:00+00:00" - ); + assert_eq!(reset.next_due_at.expect("first due"), calendar_time(0)); let fault = TransactionFault { description: "progress publication failure", table: "sqb_trigger_progress", @@ -114,8 +111,8 @@ fn given_reset_existing_ticks_when_progress_is_replayed_then_cursor_and_due_time .await .expect("advanced state"); assert_eq!( - expected[0].last_tick_for.expect("last due").to_rfc3339(), - "2020-01-03T00:00:00+00:00" + expected[0].last_tick_for.expect("last due"), + calendar_time(2) ); assert_eq!( expected[0].next_due_at.expect("future due").to_rfc3339(), @@ -140,7 +137,7 @@ fn given_reset_existing_ticks_when_progress_is_replayed_then_cursor_and_due_time assert_eq!(facts[2]["projection_json"]["last_tick_for"], json!(null)); assert_eq!( facts[3]["projection_json"]["last_tick_for"], - json!("2020-01-03T00:00:00Z") + json!(calendar_text(2)) ); commit(transaction).await.expect("source cut complete"); let mut transaction = begin(store, TransactionMode::Write) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs new file mode 100644 index 0000000000..d96b7c74e8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -0,0 +1,155 @@ +//! Expired skipped/covered ticks go; a cursor reset never recreates history older than the horizon. + +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::scheduler::_helpers::tick_retention::prune_ticks; +use crate::scheduler::main::{ + _acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger, list_ticks::list_ticks, +}; +use crate::scheduler::models::TickOutcome; +use crate::scheduler::tests::contracts::ticks::helpers::{ + append_covered_tick, calendar_time, shift_schedule, snapshot, +}; +use crate::scheduler::tests::contracts::ticks::test_types::TickRetentionTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_latest_ticks_survive_without_resurrection() + { + let test_cases = [ + TickRetentionTestCase { + description: "SQLite keeps the requested and latest ticks", + backend: Backend::Sqlite, + expected_survivors: 2, + }, + TickRetentionTestCase { + description: "PostgreSQL keeps the requested and latest ticks", + backend: Backend::Postgres, + expected_survivors: 2, + }, + ]; + for test_case in test_cases { + run(async { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/tick_retention.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("tick fixture")) + .expect("tick case"); + let shift = case["shift_days"].as_i64().expect("shift days"); + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("catch up"); + let outcomes: Vec = list_ticks(store, trigger_id, 10) + .await + .expect("ticks") + .into_iter() + .rev() + .map(|tick| tick.outcome) + .collect(); + assert_eq!( + outcomes, + [ + TickOutcome::Covered, + TickOutcome::Covered, + TickOutcome::Requested + ] + ); + let later = calendar_time(2) + TimeDelta::hours(1); + append_covered_tick(store, trigger_id, later).await; + assert_eq!(prune_ticks(store, &lease).await.expect("recent ticks"), 0); + shift_schedule(store, trigger_id, shift).await; + assert_eq!( + prune_ticks(store, &lease).await.expect("prune") as u64, + case["expected_pruned"].as_u64().expect("pruned") + ); + assert_eq!(prune_ticks(store, &lease).await.expect("idempotent"), 0); + let survivors = list_ticks(store, trigger_id, 10).await.expect("survivors"); + assert_eq!( + survivors.len(), + test_case.expected_survivors, + "{}", + test_case.description + ); + assert_eq!( + survivors + .iter() + .map(|tick| (tick.scheduled_for, tick.outcome)) + .collect::>(), + [ + (later - TimeDelta::days(shift), TickOutcome::Covered), + ( + calendar_time(2) - TimeDelta::days(shift), + TickOutcome::Requested + ), + ], + "latest and run-linked ticks survive" + ); + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + change_trigger( + store, + &audit, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: audit.actor.clone(), + reason: "Replay the retained calendar".into(), + change: TriggerChange::ResetCursor, + }, + ) + .await + .expect("reset"); + let plan = evaluate_trigger(store, &lease, trigger_id) + .await + .expect("re-evaluate") + .expect("trigger"); + assert!(plan.actions.is_empty(), "no tick older than the horizon"); + let ticks = list_ticks(store, trigger_id, 10) + .await + .expect("after reset"); + assert_eq!(ticks.len(), test_case.expected_survivors); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger") + .remove(0); + assert!(trigger.last_tick_for.is_none()); + assert_eq!( + trigger.next_due_at, + Some("2030-01-01T00:00:00Z".parse().expect("future")) + ); + assert!(ticks.iter().all(|tick| tick.scheduled_for < Utc::now())); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index 2e582f75d0..cd677556bf 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -6,6 +6,8 @@ mod test_contract; mod test_loop; #[path = "test_progress.rs"] mod test_progress; +#[path = "test_retention.rs"] +mod test_retention; #[path = "test_triggers.rs"] mod test_triggers; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index 41610c718b..6facf76fbb 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -12,7 +12,9 @@ use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; -use crate::scheduler::tests::contracts::ticks::helpers::{snapshot, trigger_audits}; +use crate::scheduler::tests::contracts::ticks::helpers::{ + calendar_text, calendar_time, snapshot, trigger_audits, +}; use crate::scheduler::tests::contracts::ticks::test_types::{ TickTransactionTestCase, TransactionFault, TriggerControlsTestCase, }; @@ -178,9 +180,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ .status, TriggerStatus::Paused ); - let time = chrono::DateTime::parse_from_rfc3339("2020-01-02T00:00:00Z") - .expect("cursor") - .to_utc(); + let time = calendar_time(1); let cursor = TriggerChangeRequest { event_id: Uuid::now_v7(), change: TriggerChange::SetCursor { time }, @@ -237,10 +237,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ .iter() .find(|row| row["event_id"] == reset_cursor.event_id.to_string()) .expect("cursor reset audit"); - assert_eq!( - cleared["before_json"]["last_tick_for"], - "2020-01-02T00:00:00Z" - ); + assert_eq!(cleared["before_json"]["last_tick_for"], calendar_text(1)); assert!(cleared["after_json"]["last_tick_for"].is_null()); assert_eq!(cleared["after_json"]["reason"], "maintenance"); close_test_store(fixture).await; @@ -310,18 +307,15 @@ fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_sur assert_eq!( serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], json!([ - "2020-01-01T00:00:00Z", - "2020-01-02T00:00:00Z", - "2020-01-03T00:00:00Z", + calendar_text(0), + calendar_text(1), + calendar_text(2), "2031-01-01T00:00:00Z" ]), "{}", test_case.description ); - assert_eq!( - trigger.next_due_at, - Some("2020-01-01T00:00:00Z".parse().expect("due")) - ); + assert_eq!(trigger.next_due_at, Some(calendar_time(0))); let lease = acquire_lease(store, Uuid::now_v7()) .await .expect("election") diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index d523cedb8e..03615e2ad3 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -35,3 +35,9 @@ impl crate::scheduler::types::RunLauncher for IdleLauncher { Ok(()) } } + +pub(crate) struct TickRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_survivors: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index c8f655b645..9db41fe26c 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -175,6 +175,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .expect("pre-sensor predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", "DELETE FROM sqb_schema_migrations WHERE name='security_audit_sink'", diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs index 6ad5ff633b..41a3310220 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_replay.rs @@ -1,5 +1,6 @@ //! Current pointers and jobs recover from retained facts, independently of trigger replay. +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use serde_json::json; use uuid::Uuid; @@ -54,8 +55,8 @@ fn given_corrupt_snapshot_projections_when_replayed_then_full_state_is_restored( }), ); input.documents.insert("triggers".into(), json!({ - "daily":{"description":"Daily","job_key":"orders","starts_at":"2020-01-01T00:00:00Z", - "schedule":{"kind":"calendar","times":["2020-01-01T00:00:00Z","2030-01-01T00:00:00Z"],"refresh_at":"2029-01-01T00:00:00Z"}} + "daily":{"description":"Daily","job_key":"orders","starts_at":calendar_text(0), + "schedule":{"kind":"calendar","times":[calendar_text(0),"2030-01-01T00:00:00Z"],"refresh_at":"2029-01-01T00:00:00Z"}} })); publish_snapshot(store, &input) .await diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index b46ed0da1e..d00ada1c15 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -92,6 +92,7 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DROP TABLE sqb_cli_login_flows", "DELETE FROM sqb_schema_migrations WHERE name='cli_login_flows'", "DROP TABLE sqb_trigger_progress", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 548327cacf..2d24669f17 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -33,6 +33,7 @@ sqb server --statistics-retention-days 30 --audit-retention-days 730 | --- | --- | --- | --- | | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | | Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | +| Skipped and covered schedule ticks | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then deleted in the background. @@ -48,6 +49,11 @@ Records kept past the horizon for these reasons are reported on stderr as `Security audit retention: retained past horizon: N` whenever the count changes. A growing count usually means the audit webhook is failing. +The scheduler deletes skipped and covered ticks older than 30 days. It keeps each trigger's latest +tick and every tick that started a run. It also never creates a tick more than 30 days old: a new +schedule, a resumed trigger or a cursor reset catches up from 30 days ago at most, so deleted +ticks never come back. + An invalid value stops the server before it opens the control database. Servers that share one control database delete at the shortest horizon among them and count only their own audit webhook, so give them the same values and webhook. diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index b46a7f7233..2cc021f980 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -25,6 +25,7 @@ sqb server --statistics-retention-days 30 --audit-retention-days 730 | --- | --- | --- | --- | | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | | Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | +| Skipped and covered schedule ticks | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then deleted in the background. @@ -40,6 +41,11 @@ Records kept past the horizon for these reasons are reported on stderr as `Security audit retention: retained past horizon: N` whenever the count changes. A growing count usually means the audit webhook is failing. +The scheduler deletes skipped and covered ticks older than 30 days. It keeps each trigger's latest +tick and every tick that started a run. It also never creates a tick more than 30 days old: a new +schedule, a resumed trigger or a cursor reset catches up from 30 days ago at most, so deleted +ticks never come back. + An invalid value stops the server before it opens the control database. Servers that share one control database delete at the shortest horizon among them and count only their own audit webhook, so give them the same values and webhook. From 169682fed2e615991dcaabb558a29d483f4712e1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:13:40 +0000 Subject: [PATCH 128/138] feat: collect old published snapshots by reference sqb server --snapshot-retention-count N (default 10) keeps the newest N published snapshots per location and target. An older snapshot is deleted only when no run, job, trigger, trigger change, location request or latest trigger progress fact references it; superseded progress facts, node definitions, SQL text and documents that nothing else shares go with it. Collection runs periodically in small batches and is safe to repeat. --- .../orchestration/snapshot_retention.json | 8 + .../sqlbuild-cli/src/argv/command_table.json | 11 ++ .../src/native_commands/_helpers/decision.rs | 30 +++- .../src/native_commands/main/_run_control.rs | 4 +- .../src/api/main/serve_api.rs | 7 +- .../src/snapshots/_helpers/mod.rs | 1 + .../src/snapshots/_helpers/retention.rs | 140 ++++++++++++++++++ .../src/snapshots/constants.rs | 5 + .../src/snapshots/main/mod.rs | 1 + .../snapshots/main/run_snapshot_retention.rs | 24 +++ .../src/snapshots/mod.rs | 1 + .../src/snapshots/models.rs | 29 ++++ .../src/store/retention_indexes.sql | 5 + .../tests/snapshots/helpers.rs | 38 +++++ .../tests/snapshots/test_contracts.rs | 2 + .../tests/snapshots/test_retention.rs | 132 +++++++++++++++++ .../tests/snapshots/test_types.rs | 7 + .../sqlbuild/references/docs/cli/server.md | 10 +- .../publication/_test_types.py | 7 + ...t_audit_retention.py => test_retention.py} | 34 ++--- website/src/content/docs/docs/cli/server.mdx | 10 +- 21 files changed, 473 insertions(+), 33 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/retention.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/run_snapshot_retention.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs rename tests/integration/src/sqlbuild/compiler/compiled_project/publication/{test_audit_retention.py => test_retention.py} (81%) diff --git a/benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json b/benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json new file mode 100644 index 0000000000..e786fafac8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json @@ -0,0 +1,8 @@ +{ + "description": "Twelve publications with distinct SQL; a run pins the first. Collection keeps the newest ten plus the pinned one, then a one-snapshot horizon keeps only the newest and the pinned one.", + "publications": 12, + "keep": 10, + "expected_collected": 1, + "shorter_keep": 1, + "expected_after_shorter": [1, 12] +} diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 97f18690fd..d352ae8208 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1165,6 +1165,17 @@ "help": "days of security audit records to keep; older records are deleted once every configured audit webhook has received them and no retained run's retry needs them (default: 400)", "group": 1 }, + { + "kind": "store", + "options": [ + "--snapshot-retention-count" + ], + "dest": "control_snapshot_retention_count", + "default": 10, + "type": "int", + "help": "published snapshots to keep per location and target; older snapshots are deleted unless a run, trigger, job or other retained record still references them (default: 10)", + "group": 1 + }, { "kind": "store", "options": ["--trusted-proxy-config"], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index a2af1a8a75..3872a753f3 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -10,6 +10,7 @@ use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScop use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; +use sqlbuild_orchestration::snapshots::models::SnapshotRetention; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::warehouse_metadata::models::StatisticsRetention; use uuid::Uuid; @@ -264,25 +265,38 @@ pub(crate) fn control_address( /// API settings are validated together before any store is opened. pub(crate) fn control_api_settings( namespace: &Namespace, -) -> Result<(ApiAuthentication, (StatisticsRetention, AuditRetention)), ControlError> { - let statistics = match retention_days(namespace, "statistics")? { +) -> Result< + ( + ApiAuthentication, + (StatisticsRetention, AuditRetention, SnapshotRetention), + ), + ControlError, +> { + let statistics = match retention_setting(namespace, "statistics-retention-days")? { Some(days) => StatisticsRetention::days(days)?, None => StatisticsRetention::default(), }; - let audit = match retention_days(namespace, "audit")? { + let audit = match retention_setting(namespace, "audit-retention-days")? { Some(days) => AuditRetention::days(days)?, None => AuditRetention::default(), }; - Ok((control_authentication(namespace)?, (statistics, audit))) + let snapshots = match retention_setting(namespace, "snapshot-retention-count")? { + Some(count) => SnapshotRetention::count(count)?, + None => SnapshotRetention::default(), + }; + Ok(( + control_authentication(namespace)?, + (statistics, audit, snapshots), + )) } /// Zero and out-of-range values name the flag before any type-specific check runs. -fn retention_days(namespace: &Namespace, kind: &str) -> Result, ControlError> { - match namespace.get(&format!("control_{kind}_retention_days")) { +fn retention_setting(namespace: &Namespace, flag: &str) -> Result, ControlError> { + match namespace.get(&format!("control_{}", flag.replace('-', "_"))) { Some(Value::Int(value)) => match value.parse::() { - Ok(days) if days > 0 => Ok(Some(days)), + Ok(setting) if setting > 0 => Ok(Some(setting)), _ => Err(ControlError::Invalid(format!( - "--{kind}-retention-days must be between 1 and 65535" + "--{flag} must be between 1 and 65535" ))), }, _ => Ok(None), diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 2fcc53cce7..6595a1f833 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -27,7 +27,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let (authentication, (statistics, audit)) = control_api_settings(namespace)?; + let (authentication, (statistics, audit, snapshots)) = control_api_settings(namespace)?; let store = open_control_store(namespace).await?; let (workers, images) = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); @@ -43,7 +43,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - (authentication, images, statistics), + (authentication, images, (statistics, snapshots)), ( move || { drop(write_output(&[( diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index e6eacbf989..d3ea9526ec 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -7,6 +7,8 @@ use crate::api::_helpers::authentication::{bearer::local_request, configure::con use crate::api::_helpers::resources::{changes, router::router}; use crate::api::main::openapi_document::openapi_document; use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, DockerImageSource}; +use crate::snapshots::main::run_snapshot_retention::run_snapshot_retention; +use crate::snapshots::models::SnapshotRetention; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; @@ -17,10 +19,10 @@ use crate::warehouse_metadata::models::StatisticsRetention; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, - (authentication, images, retention): ( + (authentication, images, (retention, snapshots)): ( ApiAuthentication, Vec, - StatisticsRetention, + (StatisticsRetention, SnapshotRetention), ), (ready, shutdown): ( impl FnOnce() + Send + 'static, @@ -60,6 +62,7 @@ pub async fn serve_api( result = async { server.await } => { result?; Ok(()) }, result = changes::relay(store.clone(), listener_changes, sender) => result, result = run_statistics_retention(&store, retention) => result, + result = run_snapshot_retention(&store, snapshots) => result, } } diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs index 8853fb96b6..12f689fdd5 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod content; pub(crate) mod preparation; pub(crate) mod publication; pub(crate) mod reading; +pub(crate) mod retention; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/retention.rs new file mode 100644 index 0000000000..9c19ece22d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/retention.rs @@ -0,0 +1,140 @@ +//! Snapshot expiry is by reference: anything a retained row points at stays, and content follows. + +use uuid::Uuid; + +use crate::snapshots::constants::{CONTENT_HASH_CHUNK, SNAPSHOT_RETENTION_BATCH}; +use crate::snapshots::models::SnapshotRetention; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +/// Beyond the newest generations, any retained reference keeps a snapshot; superseded progress goes. +const EXPIRED_SNAPSHOTS: &str = "SELECT s.snapshot_id FROM sqb_snapshots s \ + WHERE (SELECT COUNT(*) FROM sqb_snapshots newer WHERE newer.location_id=s.location_id \ + AND newer.target=s.target AND newer.generation>s.generation)>=$1 \ + AND NOT EXISTS (SELECT 1 FROM sqb_current_snapshots c WHERE c.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_runs r WHERE r.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_jobs j WHERE j.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_triggers t WHERE t.snapshot_id=s.snapshot_id \ + OR t.removed_in_snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_state_events e WHERE e.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_location_work w WHERE w.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_progress p WHERE p.snapshot_id=s.snapshot_id \ + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_progress newer WHERE newer.trigger_id=p.trigger_id \ + AND newer.storage_id>p.storage_id)) \ + ORDER BY s.published_at, s.snapshot_id LIMIT $2"; + +/// Content-addressed bodies of the expiring snapshots, removed once nothing else references them. +const CONTENT: [((&str, &str), &str, &str); 3] = [ + ( + ("sqb_snapshot_nodes", "node_hash"), + "sqb_node_definitions", + "", + ), + ( + ("sqb_snapshot_sources", "text_hash"), + "sqb_source_texts", + " AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_fingerprints f \ + WHERE f.definition_text_hash=sqb_source_texts.text_hash)", + ), + ( + ("sqb_snapshot_documents", "document_hash"), + "sqb_documents", + "", + ), +]; + +/// Batches commit independently; a repeated or interrupted pass finds only what is still expired. +pub(crate) async fn prune_snapshots( + store: &ControlStore, + retention: SnapshotRetention, +) -> Result { + let mut deleted = 0; + loop { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, rows) = query( + transaction, + EXPIRED_SNAPSHOTS, + vec![ + Parameter::Integer(retention.kept()), + Parameter::Integer(SNAPSHOT_RETENTION_BATCH as i64), + ], + ) + .await?; + let mut expired: Vec = Vec::with_capacity(rows.len()); + for row in &rows { + expired.push( + serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?, + ); + } + let transaction = delete_snapshots(transaction, &expired).await?; + commit(transaction).await?; + deleted += expired.len(); + if expired.len() < SNAPSHOT_RETENTION_BATCH { + return Ok(deleted); + } + } +} + +async fn delete_snapshots( + mut transaction: ControlTransaction, + expired: &[Uuid], +) -> Result { + if expired.is_empty() { + return Ok(transaction); + } + let placeholders = (1..=expired.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let ids: Vec = expired.iter().copied().map(Parameter::Id).collect(); + let mut hashes: Vec> = Vec::with_capacity(CONTENT.len()); + for ((table, column), _, _) in CONTENT { + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT DISTINCT {column} FROM {table} WHERE snapshot_id IN ({placeholders})"), + ids.clone(), + ) + .await?; + hashes.push( + rows.iter() + .filter_map(|row| row[column].as_str().map(str::to_owned)) + .collect(), + ); + } + for table in [ + "sqb_trigger_progress", + "sqb_snapshot_sources", + "sqb_snapshot_nodes", + "sqb_snapshot_documents", + "sqb_snapshot_changes", + "sqb_snapshots", + ] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE snapshot_id IN ({placeholders})"), + ids.clone(), + ) + .await?; + } + for (((table, column), content, elsewhere), hashes) in CONTENT.iter().zip(hashes) { + for chunk in hashes.chunks(CONTENT_HASH_CHUNK) { + let placeholders = (1..=chunk.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + (transaction, _) = query( + transaction, + &format!( + "DELETE FROM {content} WHERE {column} IN ({placeholders}) AND NOT EXISTS \ + (SELECT 1 FROM {table} n WHERE n.{column}={content}.{column}){elsewhere}" + ), + chunk.iter().cloned().map(Parameter::Text).collect(), + ) + .await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs new file mode 100644 index 0000000000..d7f3eb9d1e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -0,0 +1,5 @@ +//! Snapshot retention keeps recent publications per location and target, collected in small batches. + +pub(crate) const DEFAULT_SNAPSHOT_RETENTION_COUNT: u16 = 10; +pub(crate) const SNAPSHOT_RETENTION_BATCH: usize = 10; +pub(crate) const CONTENT_HASH_CHUNK: usize = 500; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 92e8dd726d..c39b22d88e 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -11,4 +11,5 @@ pub mod read_person_node_sql; pub(crate) mod read_snapshot_documents; pub(crate) mod read_snapshot_nodes; pub(crate) mod rebuild_projections; +pub mod run_snapshot_retention; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/run_snapshot_retention.rs b/crates/sqlbuild-orchestration/src/snapshots/main/run_snapshot_retention.rs new file mode 100644 index 0000000000..517b88ab73 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/run_snapshot_retention.rs @@ -0,0 +1,24 @@ +//! Collect unreferenced old snapshots periodically; every pass is safe to repeat. + +use std::time::Duration; + +use crate::snapshots::_helpers::retention::prune_snapshots; +use crate::snapshots::models::SnapshotRetention; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_snapshot_retention( + store: &ControlStore, + retention: SnapshotRetention, +) -> Result<(), ControlError> { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + interval.tick().await; + match prune_snapshots(store, retention).await { + Ok(_) => {} + Err(ControlError::Database(error)) if ControlError::transient_database(&error) => {} + Err(error) => return Err(error), + } + } +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs index 04ae56c307..e833f67102 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 7d8fa1bc0c..4fef7aee74 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -93,3 +93,32 @@ pub(crate) struct PreparedSnapshot { pub(crate) documents: BTreeMap)>, pub(crate) manifest_digest: String, } + +/// Installation configuration: how many recent publications to keep per location and target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SnapshotRetention { + count: u16, +} + +impl SnapshotRetention { + pub fn count(count: u16) -> Result { + if count == 0 { + return Err(crate::store::errors::ControlError::Invalid( + "snapshot retention must keep at least one snapshot".into(), + )); + } + Ok(Self { count }) + } + + pub(crate) fn kept(self) -> i64 { + i64::from(self.count) + } +} + +impl Default for SnapshotRetention { + fn default() -> Self { + Self { + count: crate::snapshots::constants::DEFAULT_SNAPSHOT_RETENTION_COUNT, + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/retention_indexes.sql b/crates/sqlbuild-orchestration/src/store/retention_indexes.sql index 1cdd99cf66..24670f1b14 100644 --- a/crates/sqlbuild-orchestration/src/store/retention_indexes.sql +++ b/crates/sqlbuild-orchestration/src/store/retention_indexes.sql @@ -1,2 +1,7 @@ CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome IN ('skipped', 'covered'); +CREATE INDEX IF NOT EXISTS sqb_runs_snapshot ON sqb_runs(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_trigger_progress_snapshot ON sqb_trigger_progress(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_snapshot_nodes_hash ON sqb_snapshot_nodes(node_hash); +CREATE INDEX IF NOT EXISTS sqb_snapshot_sources_hash ON sqb_snapshot_sources(text_hash); +CREATE INDEX IF NOT EXISTS sqb_snapshot_documents_hash ON sqb_snapshot_documents(document_hash); diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index f90e6991bf..63fc8c49ca 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -354,3 +354,41 @@ pub(super) async fn damage(store: &ControlStore) { } commit(transaction).await.expect("damage commits"); } + +/// Retained generations for one location and target, and counts that expose orphaned content. +pub(super) async fn retained_snapshot_state(store: &ControlStore, location_id: Uuid) -> Vec { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("snapshot state"); + let (transaction, rows) = crate::store::main::query::query( + transaction, + "SELECT generation FROM sqb_snapshots WHERE location_id=$1 ORDER BY generation", + vec![crate::store::models::Parameter::Id(location_id)], + ) + .await + .expect("generations"); + commit(transaction).await.expect("snapshot state read"); + rows.iter() + .map(|row| row["generation"].as_i64().expect("generation")) + .collect() +} + +/// Source texts, unreferenced source texts and trigger progress facts. +pub(super) async fn retention_counts(store: &ControlStore) -> (i64, i64, i64) { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("retention counts"); + let (transaction, rows) = crate::store::main::query::query( + transaction, + "SELECT (SELECT COUNT(*) FROM sqb_source_texts) AS texts, \ + (SELECT COUNT(*) FROM sqb_source_texts t WHERE NOT EXISTS \ + (SELECT 1 FROM sqb_snapshot_sources s WHERE s.text_hash=t.text_hash)) AS orphans, \ + (SELECT COUNT(*) FROM sqb_trigger_progress) AS progress", + vec![], + ) + .await + .expect("counts"); + commit(transaction).await.expect("retention counts read"); + let count = |name: &str| rows[0][name].as_i64().expect("count"); + (count("texts"), count("orphans"), count("progress")) +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs index d8b7b352f7..fe0cacd1ca 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -8,5 +8,7 @@ mod helpers; mod replay; #[path = "test_replay_writers.rs"] mod replay_writers; +#[path = "test_retention.rs"] +mod retention; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs new file mode 100644 index 0000000000..7033087d03 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs @@ -0,0 +1,132 @@ +//! Snapshot GC keeps the newest generations and every referenced snapshot, and collects content. + +use serde_json::json; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::rebuild_projections::rebuild_projections; +use crate::runs::tests::helpers::{request, request_run}; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::snapshots::_helpers::retention::prune_snapshots; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotRetention; +use crate::snapshots::tests::helpers::{retained_snapshot_state, retention_counts}; +use crate::snapshots::tests::test_types::SnapshotRetentionTestCase; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_twelve_snapshots_when_collected_then_ten_newest_and_run_referenced_snapshots_survive() { + let test_cases = [ + SnapshotRetentionTestCase { + description: "SQLite keeps ten newest plus the run-pinned first generation", + backend: Backend::Sqlite, + expected_generations: &[1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }, + SnapshotRetentionTestCase { + description: "PostgreSQL keeps ten newest plus the run-pinned first generation", + backend: Backend::Postgres, + expected_generations: &[1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }, + ]; + for test_case in test_cases { + run(async { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("snapshot fixture")) + .expect("snapshot case"); + let setting = |name: &str| { + SnapshotRetention::count( + u16::try_from(case[name].as_u64().expect("count")).expect("u16"), + ) + .expect("retention") + }; + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let generation_input = |generation: i64| { + let mut input = snapshot(location); + input.generation = generation; + let node = input.nodes.get_mut("orders").expect("orders node"); + node.raw_sql = Some(format!("select {generation} as order_id")); + node.compiled_sql = node.raw_sql.clone(); + node.definition = json!({"materialization": "table", "revision": generation}); + input + }; + let first = publish_snapshot(store, &generation_input(1)) + .await + .expect("publish first"); + assert!( + request_run( + store, + &request((location, first.snapshot_id), 1, 0, &["orders"]) + ) + .await + .expect("pinned run") + ); + for generation in 2..=case["publications"].as_i64().expect("publications") { + publish_snapshot(store, &generation_input(generation)) + .await + .expect("publish"); + } + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers before collection"); + let (texts, orphans, progress) = retention_counts(store).await; + assert_eq!((texts, orphans), (12, 0)); + assert_eq!( + prune_snapshots(store, setting("keep")) + .await + .expect("collect") as u64, + case["expected_collected"].as_u64().expect("collected") + ); + assert_eq!( + retained_snapshot_state(store, location).await, + test_case.expected_generations, + "{}", + test_case.description + ); + assert_eq!( + retention_counts(store).await, + (11, 0, progress - 1), + "the expired snapshot's text and superseded progress fact go with it" + ); + assert_eq!( + prune_snapshots(store, setting("keep")) + .await + .expect("idempotent"), + 0 + ); + assert!(SnapshotRetention::count(0).is_err()); + assert_eq!( + prune_snapshots(store, setting("shorter_keep")) + .await + .expect("shorter retention"), + 9 + ); + assert_eq!( + serde_json::json!(retained_snapshot_state(store, location).await), + case["expected_after_shorter"] + ); + assert_eq!(retention_counts(store).await.1, 0); + rebuild_projections(store) + .await + .expect("rebuild after collection"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("rebuilt triggers"), + triggers, + "latest progress facts alone rebuild the same trigger state" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs index 61a2c6832b..997d311376 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -20,3 +20,10 @@ pub(super) struct RecoveryTestCase { pub(super) description: &'static str, pub(super) expected_generation: i64, } + +#[derive(Debug)] +pub(super) struct SnapshotRetentionTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) expected_generations: &'static [i64], +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 2d24669f17..e161ddaf87 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -23,16 +23,17 @@ Use `sqb server --help` for service, authentication and launcher options. ## Set retention -Set how long the server keeps hourly warehouse statistics and security audit records: +Set how much history the server keeps: ```bash -sqb server --statistics-retention-days 30 --audit-retention-days 730 +sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 ``` | Data | Option | Default | Range | | --- | --- | --- | --- | | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | | Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | +| Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | | Skipped and covered schedule ticks | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then @@ -49,6 +50,11 @@ Records kept past the horizon for these reasons are reported on stderr as `Security audit retention: retained past horizon: N` whenever the count changes. A growing count usually means the audit webhook is failing. +The server keeps the newest published snapshots for each location and target. It deletes an older +snapshot only when no retained run, job, trigger, trigger change or location request uses it, and +deletes its node definitions and SQL text when no other snapshot shares them. Repeating or +interrupting this cleanup is safe. + The scheduler deletes skipped and covered ticks older than 30 days. It keeps each trigger's latest tick and every tick that started a run. It also never creates a tick more than 30 days old: a new schedule, a resumed trigger or a cursor reset catches up from 30 days ago at most, so deleted diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index d3b3efbc9e..f3e7050e53 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -272,6 +272,13 @@ class AuditHorizonTestCase: expected_retained: int +@dataclass(frozen=True) +class RetentionRejectionTestCase: + description: str + arguments: tuple[str, ...] + expected_error: str + + @dataclass(frozen=True) class StatisticsHorizonRejectionTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py similarity index 81% rename from tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py rename to tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py index b0ec7bffc4..cd62a882c6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_audit_retention.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py @@ -1,4 +1,4 @@ -"""The installed server expires security audit rows only after every configured sink has them.""" +"""The installed server validates retention settings and expires audit rows only once delivered.""" import hashlib import json @@ -15,7 +15,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( AuditHorizonTestCase, - StatisticsHorizonRejectionTestCase, + RetentionRejectionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( audit_actions, @@ -87,31 +87,31 @@ def test_given_expired_audit_rows_when_server_runs_then_only_delivered_rows_are_ @pytest.mark.parametrize( "test_case", [ - StatisticsHorizonRejectionTestCase( - "Zero days", "0", "--audit-retention-days must be between 1 and 65535" + RetentionRejectionTestCase( + "Zero audit days", + ("--audit-retention-days", "0"), + "--audit-retention-days must be between 1 and 65535", ), - StatisticsHorizonRejectionTestCase( - "Beyond the supported range", - "65536", + RetentionRejectionTestCase( + "Audit days beyond the supported range", + ("--audit-retention-days", "65536"), "--audit-retention-days must be between 1 and 65535", ), + RetentionRejectionTestCase( + "Zero snapshots", + ("--snapshot-retention-count", "0"), + "--snapshot-retention-count must be between 1 and 65535", + ), ], ids=lambda case: case.description, ) -def test_given_invalid_audit_horizon_when_server_starts_then_it_refuses_before_storage( - test_case: StatisticsHorizonRejectionTestCase, tmp_path: Path +def test_given_invalid_retention_when_server_starts_then_it_refuses_before_storage( + test_case: RetentionRejectionTestCase, tmp_path: Path ) -> None: environment: dict[str, str] = os.environ.copy() environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) result: subprocess.CompletedProcess[str] = subprocess.run( - [ - str(Path(sys.executable).with_name("sqb")), - "server", - "--port", - "0", - "--audit-retention-days", - test_case.days, - ], + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", *test_case.arguments], cwd=tmp_path, env=environment, capture_output=True, diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2cc021f980..b9b15fdcc0 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -15,16 +15,17 @@ Use `sqb server --help` for service, authentication and launcher options. ## Set retention -Set how long the server keeps hourly warehouse statistics and security audit records: +Set how much history the server keeps: ```bash -sqb server --statistics-retention-days 30 --audit-retention-days 730 +sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 ``` | Data | Option | Default | Range | | --- | --- | --- | --- | | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | | Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | +| Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | | Skipped and covered schedule ticks | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then @@ -41,6 +42,11 @@ Records kept past the horizon for these reasons are reported on stderr as `Security audit retention: retained past horizon: N` whenever the count changes. A growing count usually means the audit webhook is failing. +The server keeps the newest published snapshots for each location and target. It deletes an older +snapshot only when no retained run, job, trigger, trigger change or location request uses it, and +deletes its node definitions and SQL text when no other snapshot shares them. Repeating or +interrupting this cleanup is safe. + The scheduler deletes skipped and covered ticks older than 30 days. It keeps each trigger's latest tick and every tick that started a run. It also never creates a tick more than 30 days old: a new schedule, a resumed trigger or a cursor reset catches up from 30 days ago at most, so deleted From ecb2f9f7906229c313bb815f52406af75be0fbdf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:56:34 +0000 Subject: [PATCH 129/138] feat: require administrator approval to widen custom roles Published custom roles narrow at once. A widened definition takes effect only after a different administrator approves the activation request, or after a sole administrator explicitly confirms a self-approval. Approvals are recorded unsigned and audited. --- .../orchestration/custom_role_approval.json | 10 + clients/typescript/src/schema.d.ts | 140 +++++++++- .../authentication/password_sessions.rs | 8 + .../api/_helpers/resources/administrators.rs | 35 ++- .../sqlbuild-orchestration/src/api/models.rs | 2 + .../_helpers/administrator_roles.rs | 30 ++- .../authorization/_helpers/custom_roles.rs | 152 +++++++++++ .../src/authorization/_helpers/mod.rs | 2 + .../authorization/_helpers/role_approvals.rs | 113 ++++++++ .../authorization/main/authorize_session.rs | 16 +- .../main/decide_role_approval.rs | 51 ++++ .../src/authorization/main/mod.rs | 3 + .../main/publish_custom_roles.rs | 77 ++++++ .../main/request_role_activation.rs | 82 ++++++ .../src/credentials/models.rs | 67 ++++- .../src/declarations/_helpers/documents.rs | 1 + .../declarations/main/publish_definitions.rs | 5 +- .../src/declarations/models.rs | 1 + .../src/store/_helpers/migration.rs | 2 + .../src/store/custom_roles.sql | 43 +++ .../tests/api/test_catalog_reads.rs | 2 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_custom_roles.rs | 128 +++++++++ .../tests/api/test_external_sessions.rs | 1 + .../tests/api/test_person_location_work.rs | 2 +- .../tests/api/test_person_sessions.rs | 6 +- .../tests/api/test_types.rs | 6 + .../tests/credentials/roles/helpers.rs | 249 ++++++++++++++++++ .../tests/credentials/test_audit.rs | 7 +- .../tests/credentials/test_contracts.rs | 4 + .../tests/credentials/test_custom_roles.rs | 239 +++++++++++++++++ .../tests/credentials/test_types.rs | 15 ++ .../tests/runs/test_permissions.rs | 2 +- .../tests/sensors/test_sources.rs | 1 + .../tests/store/helpers.rs | 1 + .../compiled_project/snapshot_declarations.rs | 1 + 36 files changed, 1483 insertions(+), 23 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/custom_role_approval.json create mode 100644 crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/_helpers/role_approvals.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/decide_role_approval.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs create mode 100644 crates/sqlbuild-orchestration/src/authorization/main/request_role_activation.rs create mode 100644 crates/sqlbuild-orchestration/src/store/custom_roles.sql create mode 100644 crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_role_approval.json b/benchmarks/duckshop/edge_cases/orchestration/custom_role_approval.json new file mode 100644 index 0000000000..6ac20d433e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_role_approval.json @@ -0,0 +1,10 @@ +{ + "description": "A support role is published, approved by another administrator, widened, republished in a different action order, approved, then narrowed. Widening waits for approval; reordering keeps the same definition; narrowing applies on publish.", + "publications": [ + ["view"], + ["view", "job.run"], + ["job.run", "view"], + ["job.run"] + ], + "expected_after_narrowing": {"view": null, "job.run": "orders_support"} +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ffc7f2b06f..d48b740008 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -16,6 +16,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/admin/role-activations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestRoleActivation"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/admin/role-approvals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["decideRoleApproval"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/admin/roles": { parameters: { query?: never; @@ -1350,15 +1382,41 @@ export interface components { /** Format: uuid */ source_run_id: string; }; + /** @description The recorded activation request and the exact definition it asks to activate. */ + RoleActivation: { + definition_digest: string; + /** Format: uuid */ + request_id: string; + }; + /** @description Asks for a widened custom role definition of the current snapshot to take effect. */ + RoleActivationRequest: { + /** Format: uuid */ + location_id: string; + reason: string; + role_key: string; + target: string; + }; + /** @description An administrator's decision on an activation request; self-approval must be confirmed. */ + RoleApproval: { + confirm_self_approval?: boolean; + decision: components["schemas"]["RoleDecision"]; + reason: string; + /** Format: uuid */ + request_id: string; + }; RoleAssignmentRequest: { /** Format: uuid */ location_id?: string | null; /** Format: uuid */ principal_id: string; revoke?: boolean; - role: components["schemas"]["BuiltinRole"]; + role: components["schemas"]["RoleKey"]; target?: string | null; }; + /** @enum {string} */ + RoleDecision: "approved" | "rejected"; + /** @description A built-in role by name, or a custom role a location publishes and an administrator approves. */ + RoleKey: components["schemas"]["BuiltinRole"] | string; /** @description This is a server-owned instruction, not an arbitrary status supplied by a worker. */ RunChange: { container_id?: string | null; @@ -2140,6 +2198,86 @@ export interface operations { }; }; }; + requestRoleActivation: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RoleActivationRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RoleActivation"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + decideRoleApproval: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RoleApproval"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; assignBuiltinRole: { parameters: { query?: never; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 3fff0bfc61..a2c96c8585 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -46,6 +46,14 @@ pub(crate) fn protected(store: &ControlStore) -> Router { "/api/admin/roles", post(crate::api::_helpers::resources::administrators::assign), ) + .route( + "/api/admin/role-activations", + post(crate::api::_helpers::resources::administrators::request_activation), + ) + .route( + "/api/admin/role-approvals", + post(crate::api::_helpers::resources::administrators::decide), + ) .route( "/api/control/runs", post(crate::api::_helpers::resources::person_operations::request_job), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs index 60405dbf31..d33e0ed847 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -10,8 +10,11 @@ use crate::api::errors::ApiError; use crate::api::models::MutationResponse; use crate::authorization::main::assign_builtin_role::assign_builtin_role; use crate::authorization::main::change_administrator::change_administrator; +use crate::authorization::main::decide_role_approval::decide_role_approval; +use crate::authorization::main::request_role_activation::request_role_activation; use crate::credentials::models::{ - AdministratorAuthentication, AdministratorChange, AuditContext, RoleAssignmentRequest, + AdministratorAuthentication, AdministratorChange, AuditContext, RoleActivation, + RoleActivationRequest, RoleApproval, RoleAssignmentRequest, }; use crate::store::models::ControlStore; @@ -61,3 +64,33 @@ pub(crate) async fn assign( .await?, })) } + +#[utoipa::path(post, path = "/api/admin/role-activations", operation_id = "requestRoleActivation", + params(("x-sqb-csrf" = String, Header)), request_body = RoleActivationRequest, + responses((status = 200, body = RoleActivation), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn request_activation( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json( + request_role_activation(&store, (&context, session_credentials(&headers)), &request) + .await?, + )) +} + +#[utoipa::path(post, path = "/api/admin/role-approvals", operation_id = "decideRoleApproval", + params(("x-sqb-csrf" = String, Header)), request_body = RoleApproval, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn decide( + (State(store), Extension(context)): (State, Extension), + headers: HeaderMap, + Json(approval): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: decide_role_approval(&store, (&context, session_credentials(&headers)), &approval) + .await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index ebdda2836d..ba9d2139b1 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -15,6 +15,8 @@ use crate::api::_helpers::resources::{ #[openapi(paths( administrators::change, administrators::assign, + administrators::request_activation, + administrators::decide, audit::read, person_operations::request_job, person_operations::cancel_run, diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs index 747740c981..1e7aac8545 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs @@ -8,7 +8,7 @@ use crate::authorization::main::deny_authorization::deny; use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{ AdministratorAuthentication, AdministratorChange, AuditContext, BuiltinRole, - RoleAssignmentRequest, + RoleAssignmentRequest, RoleKey, }; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; @@ -30,7 +30,14 @@ pub(crate) async fn assign_role_in( "role target requires a location and a nonempty name".into(), )); } - if request.role == BuiltinRole::Admin { + if let RoleKey::Custom(key) = &request.role + && (key.trim().is_empty() || request.location_id.is_none()) + { + return Err(ControlError::Invalid( + "custom role assignment requires a location and a nonempty role key".into(), + )); + } + if request.role == RoleKey::Builtin(BuiltinRole::Admin) { if request.location_id.is_some() || request.target.is_some() { return deny( transaction, @@ -55,6 +62,21 @@ pub(crate) async fn assign_role_in( ) .await; } + let transaction = if let RoleKey::Custom(key) = &request.role + && !request.revoke + { + let (transaction, published) = query(transaction, + "SELECT role_key FROM sqb_role_definition_versions WHERE location_id=$1 AND role_key=$2 AND ($3 IS NULL OR target=$3) LIMIT 1", + vec![Parameter::OptionalId(request.location_id), Parameter::Text(key.clone()), Parameter::OptionalText(request.target.clone())]).await?; + if published.is_empty() { + return Err(ControlError::Invalid(format!( + "role {key} is neither a built-in role nor a custom role this location has published" + ))); + } + transaction + } else { + transaction + }; let (transaction, people) = query(transaction, "SELECT principal_id,disabled_at FROM sqb_principals WHERE principal_id=$1 AND kind='person'", vec![Parameter::Id(request.principal_id)]).await?; if people @@ -77,11 +99,11 @@ async fn assign_scoped_role( context: &AuditContext, request: &RoleAssignmentRequest, ) -> Result { - let role = serde_json::to_value(request.role) + let role = serde_json::to_value(&request.role) .map_err(|error| ControlError::Invalid(error.to_string()))?; let key = serde_json::to_string(&( request.principal_id, - request.role, + &request.role, request.location_id, &request.target, )) diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs new file mode 100644 index 0000000000..c8ade0b02f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs @@ -0,0 +1,152 @@ +//! Published custom roles narrow at once; a widened definition waits for an administrator approval. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Value, json}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::models::{Action, PublishedRole, RoleDefinition}; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +const ADMINISTRATIVE: [Action; 5] = [ + Action::LocationManage, + Action::UserManage, + Action::RoleAssign, + Action::TokenManage, + Action::AuditView, +]; + +const BUILTIN: [&str; 4] = ["viewer", "operator", "developer", "admin"]; + +fn action_name(action: Action) -> Result { + match serde_json::to_value(action) { + Ok(Value::String(name)) => Ok(name), + _ => Err(ControlError::Invalid("unnamed permission action".into())), + } +} + +pub(crate) fn decode_roles( + document: Option<&Value>, +) -> Result, ControlError> { + let roles: BTreeMap = document.map_or_else( + || Ok(BTreeMap::new()), + |document| { + serde_json::from_value(document.clone()) + .map_err(|error| ControlError::Invalid(format!("roles: {error}"))) + }, + )?; + let mut published: BTreeMap = BTreeMap::new(); + for (key, definition) in roles { + if key.trim().is_empty() + || BUILTIN.contains(&key.to_lowercase().as_str()) + || definition.description.trim().is_empty() + || definition + .actions + .iter() + .any(|action| ADMINISTRATIVE.contains(action)) + { + return Err(ControlError::Invalid(format!( + "role {key} needs a description, a key that is not a built-in role, and only non-administrative actions" + ))); + } + let mut grants: BTreeSet = BTreeSet::new(); + for action in definition.actions { + grants.insert(action_name(action)?); + } + let definition = json!({"description": definition.description, "actions": grants}); + let digest = sha256_hex(definition.to_string().as_bytes()); + published.insert( + key, + PublishedRole { + digest, + grants, + definition, + }, + ); + } + Ok(published) +} + +pub(crate) fn decode_grants(value: &Value) -> Result, ControlError> { + serde_json::from_str(value.as_str().unwrap_or("[]")) + .map_err(|error| ControlError::Schema(format!("effective role grants: {error}"))) +} + +pub(crate) fn grants_allow(value: &Value, action: Action) -> bool { + match (decode_grants(value), action_name(action)) { + (Ok(grants), Ok(name)) => grants.contains(&name), + _ => false, + } +} + +/// The effective digest and grants of one custom role, if any definition is in effect. +pub(crate) async fn effective_role( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + role_key: &str, +) -> Result<(ControlTransaction, Option<(String, BTreeSet)>), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT definition_digest,resolved_grants_json FROM sqb_effective_roles WHERE location_id=$1 AND target=$2 AND role_key=$3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(role_key.into())]).await?; + let Some(row) = rows.first() else { + return Ok((transaction, None)); + }; + let grants = decode_grants(&row["resolved_grants_json"])?; + Ok(( + transaction, + Some(( + row["definition_digest"].as_str().unwrap_or_default().into(), + grants, + )), + )) +} + +/// The definition of one custom role in the location's current snapshot. +pub(crate) async fn current_role( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + role_key: &str, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT snapshot_id FROM sqb_current_snapshots WHERE location_id=$1 AND target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ) + .await?; + let Some(snapshot_id) = rows + .first() + .map(|row| serde_json::from_value::(row["snapshot_id"].clone())) + .transpose() + .map_err(|error| ControlError::Schema(error.to_string()))? + else { + return Ok((transaction, None)); + }; + let (transaction, documents) = + read_snapshot_documents(transaction, snapshot_id, Some(&["roles"])).await?; + Ok(( + transaction, + decode_roles(documents.get("roles"))?.remove(role_key), + )) +} + +pub(crate) async fn set_effective_role( + mut transaction: ControlTransaction, + (location_id, target, role_key): (Uuid, &str, &str), + (digest, grants): (&str, &BTreeSet), +) -> Result { + let grants = + serde_json::to_string(grants).map_err(|error| ControlError::Invalid(error.to_string()))?; + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_effective_roles (location_id,target,role_key,definition_digest,resolved_grants_json,effective_since) \ + VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (location_id,target,role_key) DO UPDATE SET \ + definition_digest=excluded.definition_digest,resolved_grants_json=excluded.resolved_grants_json,effective_since=excluded.effective_since", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(role_key.into()), + Parameter::Text(digest.into()), Parameter::Text(grants), Parameter::Time(now)]).await?; + transaction.changes.insert(Change::Credentials); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs index b5107f4f02..3e82a1ac7f 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod administrator_changes; pub(crate) mod administrator_roles; pub(crate) mod administrators; +pub(crate) mod custom_roles; +pub(crate) mod role_approvals; diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/role_approvals.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/role_approvals.rs new file mode 100644 index 0000000000..5d6e8ea00c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/role_approvals.rs @@ -0,0 +1,113 @@ +//! Approval rows for custom role activation requests, and the effective role they activate. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::custom_roles::{ + current_role, effective_role, set_effective_role, +}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, RoleApproval, RoleDecision}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; + +pub(crate) async fn administrator_count( + transaction: ControlTransaction, +) -> Result<(ControlTransaction, i64), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT COUNT(*) AS administrators FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) \ + WHERE a.role_key='admin' AND a.location_id IS NULL AND a.target IS NULL AND a.revoked_at IS NULL \ + AND p.kind='person' AND p.disabled_at IS NULL", vec![]).await?; + let count = rows + .first() + .and_then(|row| row["administrators"].as_i64()) + .unwrap_or_default(); + Ok((transaction, count)) +} + +pub(crate) async fn pending_request( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Row), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.location_id,r.target,r.role_key,r.definition_digest,r.requested_by,a.approval_id FROM sqb_role_activation_requests r \ + LEFT JOIN sqb_role_approvals a ON a.request_id=r.request_id WHERE r.request_id=$1", + vec![Parameter::Id(request_id)]).await?; + match rows.into_iter().next() { + Some(row) if row["approval_id"].is_null() => Ok((transaction, row)), + Some(_) => Err(ControlError::Invalid( + "role activation request is already decided".into(), + )), + None => Err(ControlError::Invalid( + "unknown role activation request".into(), + )), + } +} + +pub(crate) async fn record_decision( + transaction: ControlTransaction, + context: &AuditContext, + approval: &RoleApproval, + (request, self_approval): (Row, bool), +) -> Result { + let location_id: Uuid = serde_json::from_value(request["location_id"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?; + let target = request["target"].as_str().unwrap_or_default(); + let role_key = request["role_key"].as_str().unwrap_or_default(); + let digest = request["definition_digest"].as_str().unwrap_or_default(); + let (transaction, before) = + effective_role(transaction, (location_id, target), role_key).await?; + let (mut transaction, role) = + current_role(transaction, (location_id, target), role_key).await?; + let approved = approval.decision == RoleDecision::Approved; + if approved && role.as_ref().is_none_or(|role| role.digest != digest) { + return Err(ControlError::Invalid( + "a newer publication superseded this role definition; request activation again".into(), + )); + } + let (approval_id, now) = (Uuid::now_v7(), transaction.now); + let decision = serde_json::to_value(approval.decision) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_role_approvals (approval_id,request_id,location_id,target,role_key,definition_digest,decision,decided_by,decided_at,reason,signature,self_approval) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,NULL,$11)", + vec![Parameter::Id(approval_id), Parameter::Id(approval.request_id), Parameter::Id(location_id), + Parameter::Text(target.into()), Parameter::Text(role_key.into()), Parameter::Text(digest.into()), + Parameter::Text(decision.as_str().unwrap_or_default().into()), Parameter::Text(context.actor.clone()), + Parameter::Time(now), Parameter::Text(approval.reason.clone()), Parameter::Boolean(self_approval)]).await?; + if let Some(role) = role.filter(|_| approved) { + transaction = set_effective_role( + transaction, + (location_id, target, role_key), + (digest, &role.grants), + ) + .await?; + } + let after = json!({"decision": decision, "definition_digest": digest, "signed": false, + "self_approval": self_approval, "requested_by": request["requested_by"]}); + transaction = append_audit( + transaction, + context, + ( + approval_id, + if approved { + "role.approved" + } else { + "role.rejected" + }, + ), + ( + json!(approval), + json!( + before + .map(|(digest, grants)| json!({"definition_digest": digest, "grants": grants})) + ), + after, + ), + ) + .await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs index 2981ec2ec2..c1f4901943 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/authorize_session.rs @@ -2,6 +2,7 @@ use serde_json::json; +use crate::authorization::_helpers::custom_roles::grants_allow; use crate::authorization::main::deny_authorization::deny; use crate::credentials::main::sessions::authenticate_session_in::authenticate_session_in; use crate::credentials::models::{ @@ -70,13 +71,16 @@ pub(crate) async fn authorize_session( .await; }; let (transaction, rows) = query(transaction, - "SELECT role_key,location_id,target FROM sqb_role_assignments WHERE principal_id=$1 AND revoked_at IS NULL ORDER BY role_key", - vec![Parameter::Id(identity.principal_id)]).await?; + "SELECT a.role_key,a.location_id,a.target,e.resolved_grants_json FROM sqb_role_assignments a \ + LEFT JOIN sqb_effective_roles e ON e.location_id=a.location_id AND e.role_key=a.role_key AND e.target=$2 \ + WHERE a.principal_id=$1 AND a.revoked_at IS NULL ORDER BY a.role_key", + vec![Parameter::Id(identity.principal_id), Parameter::OptionalText(scope.target.clone())]).await?; for row in rows { - let Ok(role) = serde_json::from_value::(row["role_key"].clone()) else { - continue; - }; - if role_allows(role, scope.action) + let allowed = serde_json::from_value::(row["role_key"].clone()).map_or_else( + |_| grants_allow(&row["resolved_grants_json"], scope.action), + |role| role_allows(role, scope.action), + ); + if allowed && (row["location_id"].is_null() || row["location_id"] == json!(scope.location_id)) && (row["target"].is_null() || row["target"] == json!(scope.target)) { diff --git a/crates/sqlbuild-orchestration/src/authorization/main/decide_role_approval.rs b/crates/sqlbuild-orchestration/src/authorization/main/decide_role_approval.rs new file mode 100644 index 0000000000..38479a463b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/decide_role_approval.rs @@ -0,0 +1,51 @@ +//! Another administrator decides a widening; a sole administrator may confirm a self-approval. + +use serde_json::json; + +use crate::authorization::_helpers::role_approvals::{ + administrator_count, pending_request, record_decision, +}; +use crate::authorization::main::authorize_session::authorize_session; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, PermissionScope, RoleApproval, +}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn decide_role_approval( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + approval: &RoleApproval, +) -> Result { + let scope = PermissionScope { + action: Action::RoleAssign, + location_id: None, + target: None, + nodes: Default::default(), + }; + let (transaction, context) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + if approval.reason.trim().is_empty() { + return Err(ControlError::Invalid("role approval needs a reason".into())); + } + let (transaction, request) = pending_request(transaction, approval.request_id).await?; + let self_approval = request["requested_by"].as_str() == Some(context.actor.as_str()); + let (transaction, administrators) = administrator_count(transaction).await?; + if self_approval && (administrators != 1 || !approval.confirm_self_approval) { + return deny( + transaction, + &context, + json!(approval), + "self-approval needs explicit confirmation and exactly one administrator; another administrator must decide", + ) + .await; + } + record_decision(transaction, &context, approval, (request, self_approval)).await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs index 6d5a0e05b9..0043c0b01f 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -2,7 +2,10 @@ pub mod assign_builtin_role; pub(crate) mod authorize_session; pub mod bootstrap_user; pub mod change_administrator; +pub mod decide_role_approval; pub(crate) mod deny_authorization; +pub(crate) mod publish_custom_roles; pub(crate) mod read_audit_events; pub mod read_security_audit; pub mod recover_administrator; +pub mod request_role_activation; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs b/crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs new file mode 100644 index 0000000000..8c001a1f65 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs @@ -0,0 +1,77 @@ +//! Record each published custom role; narrowing and removal apply now, widening waits for approval. + +use std::collections::BTreeSet; + +use serde_json::Value; + +use crate::authorization::_helpers::custom_roles::{ + decode_roles, effective_role, set_effective_role, +}; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn publish_custom_roles( + mut transaction: ControlTransaction, + record: &SnapshotRecord, + document: Option<&Value>, +) -> Result { + let roles = decode_roles(document)?; + let scope = (record.location_id, record.target.as_str()); + let rows; + (transaction, rows) = query( + transaction, + "SELECT role_key FROM sqb_effective_roles WHERE location_id=$1 AND target=$2", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into())], + ) + .await?; + for key in rows + .iter() + .filter_map(|row| row["role_key"].as_str()) + .filter(|key| !roles.contains_key(*key)) + { + (transaction, _) = query( + transaction, + "DELETE FROM sqb_effective_roles WHERE location_id=$1 AND target=$2 AND role_key=$3", + vec![ + Parameter::Id(scope.0), + Parameter::Text(scope.1.into()), + Parameter::Text(key.into()), + ], + ) + .await?; + transaction.changes.insert(Change::Credentials); + } + for (key, role) in &roles { + (transaction, _) = query(transaction, + "INSERT INTO sqb_role_definition_versions (location_id,target,role_key,definition_digest,definition_json,first_seen_snapshot_id,first_seen_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT DO NOTHING", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into()), Parameter::Text(key.clone()), + Parameter::Text(role.digest.clone()), Parameter::Text(role.definition.to_string()), + Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; + let effective; + (transaction, effective) = effective_role(transaction, scope, key).await?; + let Some((digest, grants)) = effective else { + continue; + }; + if role.grants.is_subset(&grants) { + if digest != role.digest || grants != role.grants { + transaction = set_effective_role( + transaction, + (scope.0, scope.1, key), + (&role.digest, &role.grants), + ) + .await?; + } + continue; + } + let narrowed: BTreeSet = grants.intersection(&role.grants).cloned().collect(); + if narrowed != grants { + transaction = + set_effective_role(transaction, (scope.0, scope.1, key), (&digest, &narrowed)) + .await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/request_role_activation.rs b/crates/sqlbuild-orchestration/src/authorization/main/request_role_activation.rs new file mode 100644 index 0000000000..990452cea2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/request_role_activation.rs @@ -0,0 +1,82 @@ +//! A role assigner records which widened custom role definition should take effect. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::custom_roles::{current_role, effective_role}; +use crate::authorization::main::authorize_session::authorize_session; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, PermissionScope, RoleActivation, RoleActivationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn request_role_activation( + store: &ControlStore, + (context, credentials): (&AuditContext, (&str, &str)), + request: &RoleActivationRequest, +) -> Result { + let (transaction, context) = authorize_session( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::RoleAssign, + location_id: Some(request.location_id), + target: Some(request.target.clone()), + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + if request.reason.trim().is_empty() { + return Err(ControlError::Invalid( + "role activation needs a reason".into(), + )); + } + let scope = (request.location_id, request.target.as_str()); + let (transaction, role) = current_role(transaction, scope, &request.role_key).await?; + let Some(role) = role else { + return Err(ControlError::Invalid(format!( + "role {} is not in the current published definitions", + request.role_key + ))); + }; + let (transaction, effective) = effective_role(transaction, scope, &request.role_key).await?; + if effective + .as_ref() + .is_some_and(|(_, grants)| role.grants.is_subset(grants)) + { + return Err(ControlError::Invalid(format!( + "role {} does not widen its effective definition; narrowing applies on publish", + request.role_key + ))); + } + let (request_id, now) = (Uuid::now_v7(), transaction.now); + let (transaction, _) = query(transaction, + "INSERT INTO sqb_role_activation_requests (request_id,location_id,target,role_key,definition_digest,requested_by,requested_at,reason) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + vec![Parameter::Id(request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Text(request.role_key.clone()), Parameter::Text(role.digest.clone()), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Text(request.reason.clone())]).await?; + let transaction = append_audit( + transaction, + &context, + (request_id, "role.activation_requested"), + ( + json!(request), + json!( + effective + .map(|(digest, grants)| json!({"definition_digest": digest, "grants": grants})) + ), + json!({"definition_digest": role.digest, "grants": role.grants}), + ), + ) + .await?; + commit(transaction).await?; + Ok(RoleActivation { + request_id, + definition_digest: role.digest, + }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index 2cdbc21af2..8bc9434214 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -136,6 +136,71 @@ pub enum BuiltinRole { Admin, } +/// A built-in role by name, or a custom role a location publishes and an administrator approves. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(untagged)] +pub enum RoleKey { + Builtin(BuiltinRole), + Custom(String), +} + +impl From for RoleKey { + fn from(role: BuiltinRole) -> Self { + Self::Builtin(role) + } +} + +/// One custom role entry of a published `roles` document; only non-administrative actions. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RoleDefinition { + pub(crate) description: String, + pub(crate) actions: Vec, +} + +/// A validated custom role definition with its content digest and granted action names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PublishedRole { + pub(crate) digest: String, + pub(crate) grants: std::collections::BTreeSet, + pub(crate) definition: serde_json::Value, +} + +/// Asks for a widened custom role definition of the current snapshot to take effect. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RoleActivationRequest { + pub location_id: Uuid, + pub target: String, + pub role_key: String, + pub reason: String, +} + +/// The recorded activation request and the exact definition it asks to activate. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RoleActivation { + pub request_id: Uuid, + pub definition_digest: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RoleDecision { + Approved, + Rejected, +} + +/// An administrator's decision on an activation request; self-approval must be confirmed. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RoleApproval { + pub request_id: Uuid, + pub decision: RoleDecision, + pub reason: String, + #[serde(default)] + pub confirm_self_approval: bool, +} + /// Installation-owned ambient access; never permits an administrative or execution default. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -166,7 +231,7 @@ pub(crate) struct PermissionScope { #[serde(deny_unknown_fields)] pub struct RoleAssignmentRequest { pub principal_id: Uuid, - pub role: BuiltinRole, + pub role: RoleKey, pub location_id: Option, pub target: Option, #[serde(default)] diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs index 112ceab0af..31491d736e 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -27,6 +27,7 @@ pub(crate) fn decode_definitions( let mut definitions = Definitions { jobs: document(documents, "jobs")?, groups: document(documents, "groups")?, + roles: documents.get("roles").cloned(), ..Definitions::default() }; for (name, value) in document::(documents, "triggers")? { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs index e6dfefd53b..4c181629a4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -2,6 +2,7 @@ use std::collections::BTreeSet; +use crate::authorization::main::publish_custom_roles::publish_custom_roles; use crate::declarations::_helpers::development::{pause_triggers, trigger_ids}; use crate::declarations::_helpers::publication::publish_projections; use crate::declarations::models::Definitions; @@ -12,15 +13,17 @@ use crate::store::models::ControlTransaction; pub(crate) async fn publish_definitions( transaction: ControlTransaction, snapshot: &SnapshotRecord, - definitions: Definitions, + mut definitions: Definitions, pause_new_triggers: bool, ) -> Result { + let roles = definitions.roles.take(); let (transaction, previous) = if pause_new_triggers { trigger_ids(transaction, (snapshot.location_id, &snapshot.target)).await? } else { (transaction, BTreeSet::new()) }; let transaction = publish_projections(transaction, snapshot, definitions).await?; + let transaction = publish_custom_roles(transaction, snapshot, roles.as_ref()).await?; if pause_new_triggers { pause_triggers( transaction, diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 409d76b88c..d6e9b564c1 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -92,6 +92,7 @@ pub struct Definitions { pub groups: BTreeMap, pub triggers: BTreeMap, pub sensors: BTreeMap, + pub roles: Option, } /// Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 4575c78ee5..0aae15d12e 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -43,6 +43,7 @@ pub(crate) const CHECK_OBSERVATIONS: &str = include_str!("../check_observations. pub(crate) const TRIGGER_PROGRESS: &str = include_str!("../trigger_progress.sql"); pub(crate) const CLI_LOGIN_FLOWS: &str = include_str!("../cli_login_flows.sql"); pub(crate) const RETENTION_INDEXES: &str = include_str!("../retention_indexes.sql"); +pub(crate) const CUSTOM_ROLES: &str = include_str!("../custom_roles.sql"); pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("location_instances", LOCATION_INSTANCES), @@ -78,6 +79,7 @@ pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("trigger_progress", TRIGGER_PROGRESS), ("cli_login_flows", CLI_LOGIN_FLOWS), ("retention_indexes", RETENTION_INDEXES), + ("custom_roles", CUSTOM_ROLES), ]; pub(crate) fn migration_digests(store: &ControlStore) -> Vec { diff --git a/crates/sqlbuild-orchestration/src/store/custom_roles.sql b/crates/sqlbuild-orchestration/src/store/custom_roles.sql new file mode 100644 index 0000000000..221af0dae7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/custom_roles.sql @@ -0,0 +1,43 @@ +CREATE TABLE IF NOT EXISTS sqb_role_definition_versions ( + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + role_key TEXT NOT NULL, + definition_digest TEXT NOT NULL, + definition_json TEXT NOT NULL, + first_seen_snapshot_id {uuid} NOT NULL, + first_seen_at {time} NOT NULL, + PRIMARY KEY (location_id, target, role_key, definition_digest) +); +CREATE TABLE IF NOT EXISTS sqb_role_activation_requests ( + request_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + definition_digest TEXT NOT NULL, + requested_by TEXT NOT NULL, + requested_at {time} NOT NULL, + reason TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS sqb_role_approvals ( + approval_id {uuid} PRIMARY KEY, + request_id {uuid} NOT NULL UNIQUE REFERENCES sqb_role_activation_requests(request_id), + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + definition_digest TEXT NOT NULL, + decision TEXT NOT NULL, + decided_by TEXT NOT NULL, + decided_at {time} NOT NULL, + reason TEXT NOT NULL, + signature TEXT, + self_approval BOOLEAN NOT NULL +); +CREATE TABLE IF NOT EXISTS sqb_effective_roles ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + definition_digest TEXT NOT NULL, + resolved_grants_json TEXT NOT NULL, + effective_since {time} NOT NULL, + PRIMARY KEY (location_id, target, role_key) +); diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index 07b7264348..aca36dbda6 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -162,7 +162,7 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp )); let mut assignment = RoleAssignmentRequest { principal_id: person, - role: BuiltinRole::Viewer, + role: BuiltinRole::Viewer.into(), location_id: Some(location), target: Some("development".into()), revoke: false, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index c126023cfc..fb3371f9f2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -36,6 +36,8 @@ mod test_changes; mod test_cli_login; #[path = "test_contract.rs"] mod test_contract; +#[path = "test_custom_roles.rs"] +mod test_custom_roles; #[path = "test_development.rs"] mod test_development; #[path = "test_external_sessions.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs b/crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs new file mode 100644 index 0000000000..5f579eaba8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs @@ -0,0 +1,128 @@ +//! Custom role assignment, activation and approval work over HTTP with session CSRF. + +use reqwest::header; +use serde_json::{Value, json}; + +use crate::api::models::ApiAuthentication; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::CustomRoleApiTestCase; +use crate::credentials::models::Action; +use crate::credentials::tests::role_helpers::{SUPPORT_ROLE, granting_role, role_fixture}; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_http_sessions_when_custom_role_is_assigned_requested_and_approved_then_only_another_administrator_activates_it() + { + let test_cases = [ + CustomRoleApiTestCase { + description: "SQLite custom role approval over HTTP", + backend: Backend::Sqlite, + expected_statuses: [400, 200, 200, 403, 200], + }, + CustomRoleApiTestCase { + description: "PostgreSQL custom role approval over HTTP", + backend: Backend::Postgres, + expected_statuses: [400, 200, 200, 403, 200], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store, 2).await; + let api = start_api_with_authentication( + store, + ApiAuthentication { + password_origin: Some("https://control.example.com".into()), + ..ApiAuthentication::default() + }, + ) + .await; + let client = reqwest::Client::new(); + let post = |administrator: usize, path: &str, body: Value| { + let session = &roles.administrators[administrator]; + client + .post(format!("{}{path}", api.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .header( + header::COOKIE, + format!("__Host-sqb-session={}", session.bearer), + ) + .header("x-sqb-csrf", &session.csrf) + .json(&body) + .send() + }; + let misspelt = post( + 0, + "/api/admin/roles", + json!({"principal_id": roles.member_id, "role": "Operator", + "location_id": roles.location_id, "target": "development"}), + ) + .await + .expect("misspelt role") + .status() + .as_u16(); + let assignment = post( + 0, + "/api/admin/roles", + json!({"principal_id": roles.member_id, "role": SUPPORT_ROLE, + "location_id": roles.location_id, "target": "development"}), + ) + .await + .expect("assign"); + let assignment_status = assignment.status().as_u16(); + let activation = post( + 0, + "/api/admin/role-activations", + json!({"location_id": roles.location_id, "target": "development", + "role_key": SUPPORT_ROLE, "reason": "Support answers order questions"}), + ) + .await + .expect("request"); + let activation_status = activation.status().as_u16(); + let activation: Value = activation.json().await.expect("activation body"); + let approval = |confirm: bool| { + json!({"request_id": activation["request_id"], "decision": "approved", + "reason": "Reviewed", "confirm_self_approval": confirm}) + }; + let refused = post(0, "/api/admin/role-approvals", approval(true)) + .await + .expect("self approval") + .status() + .as_u16(); + let approved = post(1, "/api/admin/role-approvals", approval(false)) + .await + .expect("approval") + .status() + .as_u16(); + assert_eq!( + [ + misspelt, + assignment_status, + activation_status, + refused, + approved + ], + test_case.expected_statuses, + "{}", + test_case.description + ); + assert_eq!( + activation["definition_digest"].as_str().map(str::len), + Some(64) + ); + assert_eq!( + granting_role(store, &roles, Action::View).await.as_deref(), + Some(SUPPORT_ROLE) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs index 055f5cd23f..1ad9760fca 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -94,6 +94,7 @@ fn given_federated_people_when_administrators_and_sessions_change_then_live_auth .expect("populated predecessor"); for sql in [ "DROP TABLE sqb_external_sessions", + "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='external_sessions'", ] { diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index 07a08d5761..dbde2fe30c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -88,7 +88,7 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced .expect("publish"); let assignment = RoleAssignmentRequest { principal_id: person, - role: BuiltinRole::Developer, + role: BuiltinRole::Developer.into(), location_id: Some(location), target: Some("development".into()), revoke: false, diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index 03979f0fe9..135bc92a77 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -185,7 +185,7 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ (&context, (&admin_session.bearer, &admin_session.csrf)), &RoleAssignmentRequest { principal_id: person, - role, + role: role.into(), location_id: None, target: None, revoke: false, @@ -346,7 +346,7 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ (&context, (&admin_session.bearer, &admin_session.csrf)), &RoleAssignmentRequest { principal_id: person, - role, + role: role.into(), location_id: None, target: None, revoke: true, @@ -418,7 +418,7 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ (&context, (&admin_session.bearer, &admin_session.csrf)), &RoleAssignmentRequest { principal_id: person, - role: crate::credentials::models::BuiltinRole::Developer, + role: crate::credentials::models::BuiltinRole::Developer.into(), location_id: Some(location), target: Some("development".into()), revoke: false, diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 09e69bb3d9..0dce36c35b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -111,3 +111,9 @@ pub(crate) struct CliLoginTestCase { pub(crate) backend: Backend, pub(crate) expected_exchanges: usize, } + +pub(crate) struct CustomRoleApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_statuses: [u16; 5], +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs new file mode 100644 index 0000000000..75989abd3f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs @@ -0,0 +1,249 @@ +//! Real people, sessions and published custom roles for role approval contracts. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, authorize_session::authorize_session, + bootstrap_user::bootstrap_user, +}; +use crate::authorization::main::{ + decide_role_approval::decide_role_approval, request_role_activation::request_role_activation, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + Action, AdministratorAuthentication, AuditContext, BuiltinRole, DefaultRole, IssuedSession, + PasswordAuthority, PasswordLogin, PermissionScope, PrincipalKind, RoleAssignmentRequest, + RoleKey, +}; +use crate::credentials::models::{ + RoleActivation, RoleActivationRequest, RoleApproval, RoleDecision, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::test_types::RoleFixture; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) const SUPPORT_ROLE: &str = "orders_support"; + +async fn signed_in( + store: &ControlStore, + context: &AuditContext, + (principal, name): (Uuid, &str), +) -> IssuedSession { + set_password( + store, + context, + PasswordAuthority::Operator(principal), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid login"); + }; + session +} + +pub(crate) fn credentials(session: &IssuedSession) -> (&str, &str) { + (session.bearer.as_str(), session.csrf.as_str()) +} + +async fn assign( + store: &ControlStore, + fixture: (&AuditContext, &IssuedSession), + request: RoleAssignmentRequest, +) { + assert!( + assign_builtin_role( + store, + (fixture.0, credentials(fixture.1)), + &request, + &AdministratorAuthentication::Password + ) + .await + .expect("role assignment") + ); +} + +/// Publish one generation whose `roles` document grants the support role these actions. +pub(crate) async fn publish_support_role( + store: &ControlStore, + location_id: Uuid, + (generation, actions): (i64, Value), +) { + let mut input = snapshot(location_id); + input.generation = generation; + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", "actions": actions}}), + ); + publish_snapshot(store, &input) + .await + .expect("publish roles"); +} + +/// Administrators by count, then a member holding the published, still unapproved support role. +pub(crate) async fn role_fixture(store: &ControlStore, administrators: usize) -> RoleFixture { + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("local peer")), + }; + let first = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let mut sessions = vec![signed_in(store, &context, (first, "orders-admin")).await]; + for index in 1..administrators { + let name = format!("fulfillment-admin-{index}"); + let principal = create_principal(store, &context, (&name, PrincipalKind::Person)) + .await + .expect("administrator"); + let session = signed_in(store, &context, (principal, &name)).await; + assign( + store, + (&context, &sessions[0]), + RoleAssignmentRequest { + principal_id: principal, + role: BuiltinRole::Admin.into(), + location_id: None, + target: None, + revoke: false, + }, + ) + .await; + sessions.push(session); + } + let member = create_principal(store, &context, ("support-person", PrincipalKind::Person)) + .await + .expect("member"); + let member_session = signed_in(store, &context, (member, "support-person")).await; + let location_id = create_location(store, "orders", "orders-admin") + .await + .expect("location"); + publish_support_role(store, location_id, (1, json!(["view"]))).await; + assign( + store, + (&context, &sessions[0]), + RoleAssignmentRequest { + principal_id: member, + role: RoleKey::Custom(SUPPORT_ROLE.into()), + location_id: Some(location_id), + target: None, + revoke: false, + }, + ) + .await; + RoleFixture { + context, + location_id, + administrators: sessions, + member: member_session, + member_id: member, + } +} + +/// The role that grants the member this action in the fixture location, or `None` when denied. +pub(crate) async fn granting_role( + store: &ControlStore, + fixture: &RoleFixture, + action: Action, +) -> Option { + let scope = PermissionScope { + action, + location_id: Some(fixture.location_id), + target: Some("development".into()), + nodes: Default::default(), + }; + let decision = authorize_session( + begin(store, TransactionMode::Write).await.expect("begin"), + (&fixture.context, credentials(&fixture.member)), + &scope, + DefaultRole::None, + ) + .await; + assert!( + decision + .as_ref() + .err() + .is_none_or(|error| matches!(error, ControlError::Denied { .. })), + "authorization either grants or denies" + ); + decision.ok().map(|(_, context)| context.granting_role) +} + +/// Role approval audit facts in commit order: action and recorded outcome. +pub(crate) async fn role_audit(store: &ControlStore) -> Vec<(String, Value)> { + let transaction = begin(store, TransactionMode::Read).await.expect("read"); + let (transaction, rows) = query( + transaction, + "SELECT action, CAST(after_json AS TEXT) AS after FROM sqb_security_audit \ + WHERE action IN ('role.activation_requested','role.approved','role.rejected') ORDER BY storage_id", + vec![], + ) + .await + .expect("role audit"); + commit(transaction).await.expect("commit"); + rows.iter() + .map(|row| { + ( + row["action"].as_str().unwrap_or_default().into(), + serde_json::from_str(row["after"].as_str().unwrap_or("null")).expect("after json"), + ) + }) + .collect() +} + +pub(crate) async fn request( + store: &ControlStore, + fixture: &RoleFixture, + requester: usize, +) -> RoleActivation { + request_role_activation( + store, + ( + &fixture.context, + credentials(&fixture.administrators[requester]), + ), + &RoleActivationRequest { + location_id: fixture.location_id, + target: "development".into(), + role_key: SUPPORT_ROLE.into(), + reason: "Support answers order questions".into(), + }, + ) + .await + .expect("activation request") +} + +pub(crate) async fn approve( + store: &ControlStore, + fixture: &RoleFixture, + (approver, activation, confirm): (usize, &RoleActivation, bool), +) -> Result { + decide_role_approval( + store, + ( + &fixture.context, + credentials(&fixture.administrators[approver]), + ), + &RoleApproval { + request_id: activation.request_id, + decision: RoleDecision::Approved, + reason: "Reviewed the support grants".into(), + confirm_self_approval: confirm, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 6734e12289..5cdb5276a7 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -110,7 +110,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ ), &RoleAssignmentRequest { principal_id: person, - role, + role: role.into(), location_id: None, target: None, revoke: false, @@ -139,6 +139,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .expect("predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", @@ -345,7 +346,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ ), &RoleAssignmentRequest { principal_id: person, - role: BuiltinRole::Admin, + role: BuiltinRole::Admin.into(), location_id: None, target: None, revoke: false, @@ -364,7 +365,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ ), &RoleAssignmentRequest { principal_id: admin, - role: BuiltinRole::Admin, + role: BuiltinRole::Admin.into(), location_id: None, target: None, revoke: true, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs index bcbd4651cd..d8c431a139 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -4,6 +4,8 @@ pub(crate) mod helpers; #[path = "locations/helpers.rs"] pub(crate) mod location_helpers; +#[path = "roles/helpers.rs"] +pub(crate) mod role_helpers; #[path = "test_administrators.rs"] mod test_administrators; #[path = "test_audit.rs"] @@ -12,6 +14,8 @@ mod test_audit; mod test_audit_retention; #[path = "test_contract.rs"] mod test_contract; +#[path = "test_custom_roles.rs"] +mod test_custom_roles; #[path = "passwords/test_contract.rs"] mod test_passwords; #[path = "test_recovery.rs"] diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs b/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs new file mode 100644 index 0000000000..aa7f96c554 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs @@ -0,0 +1,239 @@ +//! Widened custom roles take effect only through another administrator or a confirmed sole one. + +use serde_json::json; + +use crate::credentials::models::Action; +use crate::credentials::tests::role_helpers::{ + SUPPORT_ROLE, approve, granting_role, publish_support_role, request, role_audit, role_fixture, +}; +use crate::credentials::tests::test_types::CustomRoleTestCase; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_widened_custom_role_when_unapproved_then_new_grants_stay_refused() { + let test_cases = [ + CustomRoleTestCase { + description: "SQLite refuses an unapproved widening", + backend: Backend::Sqlite, + expected_role: None, + }, + CustomRoleTestCase { + description: "PostgreSQL refuses an unapproved widening", + backend: Backend::Postgres, + expected_role: None, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store, 2).await; + let activation = request(store, &roles, 0).await; + assert_eq!( + granting_role(store, &roles, Action::View).await.as_deref(), + test_case.expected_role, + "{}: a request alone grants nothing", + test_case.description + ); + approve(store, &roles, (1, &activation, false)) + .await + .expect("first approval"); + publish_support_role(store, roles.location_id, (2, json!(["view", "job.run"]))).await; + assert_eq!( + granting_role(store, &roles, Action::JobRun) + .await + .as_deref(), + test_case.expected_role, + "{}: the widened action waits for approval", + test_case.description + ); + assert_eq!( + granting_role(store, &roles, Action::View).await.as_deref(), + Some(SUPPORT_ROLE), + "{}: unchanged grants stay effective", + test_case.description + ); + assert!(matches!( + approve(store, &roles, (1, &activation, false)).await, + Err(ControlError::Invalid(_)) + )); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_widened_custom_role_when_another_administrator_approves_then_it_applies_and_narrowing_is_immediate() + { + let test_cases = [ + CustomRoleTestCase { + description: "SQLite applies another administrator's approval", + backend: Backend::Sqlite, + expected_role: Some(SUPPORT_ROLE), + }, + CustomRoleTestCase { + description: "PostgreSQL applies another administrator's approval", + backend: Backend::Postgres, + expected_role: Some(SUPPORT_ROLE), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/custom_role_approval.json", + ); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("role fixture")) + .expect("role case"); + let publication = |index: usize| case["publications"][index].clone(); + let roles = role_fixture(store, 2).await; + let first = request(store, &roles, 0).await; + assert!( + approve(store, &roles, (1, &first, false)) + .await + .expect("approve") + ); + publish_support_role(store, roles.location_id, (2, publication(1))).await; + let widened = request(store, &roles, 0).await; + publish_support_role(store, roles.location_id, (3, publication(2))).await; + assert!( + approve(store, &roles, (1, &widened, false)) + .await + .expect("approve widening") + ); + assert_eq!( + granting_role(store, &roles, Action::JobRun) + .await + .as_deref(), + test_case.expected_role, + "{}", + test_case.description + ); + publish_support_role(store, roles.location_id, (4, publication(3))).await; + assert_eq!( + json!({ + "view": granting_role(store, &roles, Action::View).await, + "job.run": granting_role(store, &roles, Action::JobRun).await, + }), + case["expected_after_narrowing"], + "{}: narrowing applies on publish", + test_case.description + ); + let audit = role_audit(store).await; + assert_eq!(audit.len(), 4); + assert_eq!( + ( + audit[3].0.as_str(), + &audit[3].1["signed"], + &audit[3].1["self_approval"] + ), + ("role.approved", &json!(false), &json!(false)) + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_sole_administrator_when_self_approval_is_confirmed_then_it_applies_and_is_audited() { + let test_cases = [ + CustomRoleTestCase { + description: "SQLite sole administrator confirms a self-approval", + backend: Backend::Sqlite, + expected_role: Some(SUPPORT_ROLE), + }, + CustomRoleTestCase { + description: "PostgreSQL sole administrator confirms a self-approval", + backend: Backend::Postgres, + expected_role: Some(SUPPORT_ROLE), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store, 1).await; + let activation = request(store, &roles, 0).await; + assert!(matches!( + approve(store, &roles, (0, &activation, false)).await, + Err(ControlError::Denied { .. }) + )); + assert!( + approve(store, &roles, (0, &activation, true)) + .await + .expect("confirmed") + ); + assert_eq!( + granting_role(store, &roles, Action::View).await.as_deref(), + test_case.expected_role, + "{}", + test_case.description + ); + let audit = role_audit(store).await; + assert_eq!( + ( + audit[1].0.as_str(), + &audit[1].1["signed"], + &audit[1].1["self_approval"] + ), + ("role.approved", &json!(false), &json!(true)), + "{}: the approval is audited as an unsigned self-approval", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_two_administrators_when_requester_self_approves_then_it_is_refused() { + let test_cases = [ + CustomRoleTestCase { + description: "SQLite refuses self-approval with another administrator", + backend: Backend::Sqlite, + expected_role: None, + }, + CustomRoleTestCase { + description: "PostgreSQL refuses self-approval with another administrator", + backend: Backend::Postgres, + expected_role: None, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store, 2).await; + let activation = request(store, &roles, 0).await; + assert!(matches!( + approve(store, &roles, (0, &activation, true)).await, + Err(ControlError::Denied { .. }) + )); + assert_eq!( + granting_role(store, &roles, Action::View).await.as_deref(), + test_case.expected_role, + "{}", + test_case.description + ); + assert_eq!(role_audit(store).await.len(), 1); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs index 00c52cf199..7eb14ccc5b 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -40,3 +40,18 @@ pub(crate) struct AuditRetentionTestCase { pub(crate) backend: Backend, pub(crate) expected_retained: i64, } + +pub(crate) struct CustomRoleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_role: Option<&'static str>, +} + +/// Administrators by sign-in order, and a member holding the published support role. +pub(crate) struct RoleFixture { + pub(crate) context: crate::credentials::models::AuditContext, + pub(crate) location_id: uuid::Uuid, + pub(crate) administrators: Vec, + pub(crate) member: crate::credentials::models::IssuedSession, + pub(crate) member_id: uuid::Uuid, +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index b1cf015f69..330959d653 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -150,7 +150,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi ); let assignment = RoleAssignmentRequest { principal_id: person, - role: test_case.role, + role: test_case.role.into(), location_id: None, target: None, revoke: false, diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 9db41fe26c..87a83c4487 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -175,6 +175,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .expect("pre-sensor predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", "DROP TABLE sqb_security_audit_deliveries", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index d00ada1c15..baa49874ed 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -92,6 +92,7 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DROP TABLE sqb_cli_login_flows", "DELETE FROM sqb_schema_migrations WHERE name='cli_login_flows'", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index d30b0e4bff..4a551af132 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -201,6 +201,7 @@ pub(crate) fn compile_declarations( groups, triggers, sensors, + .. } = compile_sensors( py, (project_dir, &input, graph), From bb8406f2d863307f605f21213bc996996053c999 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:12:17 +0000 Subject: [PATCH 130/138] fix: bound each schedule tick cleanup pass A cleanup pass now deletes at most ten batches. A longer backlog resumes on the next scheduler cycle, after the lease renews, so a large backlog of old covered ticks cannot outlast the scheduler lease. --- .../orchestration/tick_retention.json | 9 ++++++-- .../src/scheduler/_helpers/tick_retention.rs | 22 +++++++++++++------ .../src/scheduler/constants.rs | 2 ++ .../tests/scheduler/ticks/test_retention.rs | 14 ++++++++++++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json index b5f01f704b..73d3ff4472 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json +++ b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json @@ -1,6 +1,11 @@ { - "description": "Covered ticks age past the tick horizon and are pruned; the requested tick and the latest covered tick stay; the operator then resets the cursor.", + "description": "A backlog of covered ticks ages past the tick horizon. One cleanup pass stops at its batch cap and the next pass finishes; the requested tick and the latest covered tick stay; the operator then resets the cursor.", "shift_days": 40, + "backlog_ticks": 1000, + "expected_first_pass": 1000, "expected_pruned": 2, - "expected_survivors": ["covered", "requested"] + "expected_survivors": [ + "covered", + "requested" + ] } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs index d3d5b21f08..db668918c7 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs @@ -6,7 +6,8 @@ use chrono::{DateTime, TimeDelta, Utc}; use uuid::Uuid; use crate::scheduler::constants::{ - TICK_RETENTION_BATCH, TICK_RETENTION_DAYS, TICK_RETENTION_INTERVAL_SECONDS, + TICK_RETENTION_BATCH, TICK_RETENTION_BATCHES_PER_PASS, TICK_RETENTION_DAYS, + TICK_RETENTION_INTERVAL_SECONDS, }; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; @@ -28,7 +29,7 @@ pub(crate) fn evaluation_start( Ok(start.max(now - TimeDelta::days(TICK_RETENTION_DAYS))) } -/// Run the leader's tick cleanup at most once per interval; returns when it last ran. +/// Run the leader's tick cleanup at most once per interval; a capped pass runs again next cycle. pub(crate) async fn prune_ticks_when_due( store: &ControlStore, lease: &SchedulerLease, @@ -38,19 +39,25 @@ pub(crate) async fn prune_ticks_when_due( match pruned_at { Some(at) if at.elapsed() < interval => Ok(Some(at)), _ => { - prune_ticks(store, lease).await?; - Ok(Some(Instant::now())) + let deleted = prune_ticks(store, lease).await?; + Ok( + if deleted == TICK_RETENTION_BATCH * TICK_RETENTION_BATCHES_PER_PASS { + pruned_at + } else { + Some(Instant::now()) + }, + ) } } } -/// Requested, failed and deferred ticks stay; so does any tick a run row points at. +/// One bounded pass; requested, failed and deferred ticks stay, as does any tick a run points at. pub(crate) async fn prune_ticks( store: &ControlStore, lease: &SchedulerLease, ) -> Result { let mut deleted = 0; - loop { + for _ in 0..TICK_RETENTION_BATCHES_PER_PASS { let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; let cutoff = transaction.now - TimeDelta::days(TICK_RETENTION_DAYS); @@ -76,7 +83,8 @@ pub(crate) async fn prune_ticks( commit(check_lease(transaction, lease).await?).await?; deleted += rows.len(); if rows.len() < TICK_RETENTION_BATCH { - return Ok(deleted); + break; } } + Ok(deleted) } diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs index b8c4f3b521..644f3424d0 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/constants.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -9,4 +9,6 @@ pub const MAX_TICKS_PER_BATCH: usize = 10_000; /// Skipped and covered ticks older than this are deleted, and no tick older than it is created. pub(crate) const TICK_RETENTION_DAYS: i64 = 30; pub(crate) const TICK_RETENTION_BATCH: usize = 100; +/// Batches per cleanup pass; a longer backlog resumes next cycle, after the lease renews. +pub(crate) const TICK_RETENTION_BATCHES_PER_PASS: usize = 10; pub(crate) const TICK_RETENTION_INTERVAL_SECONDS: u64 = 60; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs index d96b7c74e8..0ead6f6bf1 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -82,9 +82,23 @@ fn given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_ ] ); let later = calendar_time(2) + TimeDelta::hours(1); + let backlog = case["backlog_ticks"].as_i64().expect("backlog"); + for second in 1..=backlog { + append_covered_tick( + store, + trigger_id, + calendar_time(2) + TimeDelta::seconds(second), + ) + .await; + } append_covered_tick(store, trigger_id, later).await; assert_eq!(prune_ticks(store, &lease).await.expect("recent ticks"), 0); shift_schedule(store, trigger_id, shift).await; + assert_eq!( + prune_ticks(store, &lease).await.expect("bounded pass") as u64, + case["expected_first_pass"].as_u64().expect("first pass"), + "one pass stops at its batch cap so the lease renews before the rest" + ); assert_eq!( prune_ticks(store, &lease).await.expect("prune") as u64, case["expected_pruned"].as_u64().expect("pruned") From 5601aefc318f065dcabed555cfd9599d68123927 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 04:36:34 +0000 Subject: [PATCH 131/138] test: keep scheduled worker fixtures inside the tick horizon Worker and recovery fixtures used an annual schedule starting in 2000 and expected an immediate catch-up run. The scheduler no longer creates ticks older than the 30-day tick horizon, so the latest annual occurrence was never ticked and each test waited out its 30-second run deadline. A daily schedule keeps one catch-up run inside the horizon. --- .../edge_cases/orchestration/attached_checks.json | 2 +- .../orchestration/local_recovery_orders.py | 2 +- .../compiled_project/publication/helpers.py | 2 +- .../compiled_project/publication/test_calendar.py | 2 +- .../publication/test_docker_worker.py | 2 +- .../compiled_project/publication/test_rerun.py | 2 +- .../compiled_project/publication/test_worker.py | 14 +++++++------- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json index 79e6d5d12c..f111fc629a 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -3,6 +3,6 @@ "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) AS seed(i) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", - "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", + "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py index 92f64e9d81..6724f992aa 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py @@ -7,5 +7,5 @@ description="Build orders under a shared concurrency limit.", select="orders", concurrency_groups=("orders_limit",), - schedule=schedule(cron="0 0 1 1 *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 286d02156c..3399cd9cf2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -637,7 +637,7 @@ def scheduled_worker( " assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' 'starts_at="2000-01-01T00:00:00Z"))\n' ), } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 52fabe8463..9f3a98f241 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -275,7 +275,7 @@ def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not | { "orchestration/orders.py": ( "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" - "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" "def deliveries(window):\n" f" marker = Path({str(evaluated)!r})\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index faeb3c93cb..253eda3323 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -59,7 +59,7 @@ " assert name not in os.environ, name\n" 'annual_orders = job("annual_orders", select="orders", ' 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' 'starts_at="2000-01-01T00:00:00Z"))\n' ) }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py index f7edf07364..1d1193a3a9 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -65,7 +65,7 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ "marts = job('marts', description='Build totals.', select='totals')\n" "inventory = job('inventory', description='Build inventory.', select='inventory')\n" "annual_orders = job_group('annual_orders', description='Retry boundaries.', " - "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 1 1 *', " + "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 * * *', " "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" ), }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 80d8941720..e0fd192e5b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -156,7 +156,7 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' "jobs=[marts, staging], " - 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' 'starts_at="2000-01-01T00:00:00Z"))\n' ), } @@ -231,10 +231,10 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "concurrency_groups=('orders_limit',))\n" "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " "concurrency_groups=('orders_limit',), " - "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "annual_inventory = job_group('annual_inventory', description='Inventory group.', " "jobs=[inventory], concurrency_groups=('inventory_limit',), " - "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" ), }, store_setup=( @@ -437,7 +437,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "from sqlbuild.orchestration import job, schedule\n" "annual_orders = job('annual_orders', select='orders', description='Build orders.', " "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " - "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "schedule=schedule(cron='0 0 * * *', timezone='UTC', " "starts_at='2000-01-01T00:00:00Z'))\n" ), }, @@ -719,7 +719,7 @@ def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_fai "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" "source = job('source_orders', select='orders', description='Source orders.')\n" "source_group = job_group('source_group', description='Source group.', jobs=[source], " - "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" "on_group = sensor('on_group', description='Refresh after group completion.', " "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " @@ -774,7 +774,7 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" "source = job('source_orders', select='orders', description='Source orders.')\n" "source_group = job_group('source_group', description='Source group.', jobs=[source], " - "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" "on_node = sensor('on_node', description='React to actual orders rebuilds.', " "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " @@ -858,7 +858,7 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s "orders = job('orders', description='Long query.', select='orders')\n" "totals = job('totals', description='Waiting query.', select='totals')\n" "annual_orders = job_group('annual_orders', description='Cancellable group.', " - "jobs=[totals, orders], schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "jobs=[totals, orders], schedule=schedule(cron='0 0 * * *', timezone='UTC', " "starts_at='2000-01-01T00:00:00Z'))\n" ), }, From 40fbc759f5d25ed0d127098565aa7091400740a9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:11:04 +0000 Subject: [PATCH 132/138] fix: give latest catch-up one tick older than the tick horizon With catch_up="latest", a trigger always gets exactly one tick for its latest occurrence, even when that occurrence is older than the 30-day tick horizon. A yearly schedule published with a start years back therefore catches up once, as before the horizon, instead of waiting for its next occurrence. The horizon still bounds scheduler work and still stops re-deciding old ticks: nothing older than the latest occurrence is created, an occurrence that already has a tick is not decided again after a cursor reset, and bounded catch-up keeps the horizon clamp. The worker fixtures keep their daily schedule. --- .../orchestration/tick_retention.json | 6 +- .../src/scheduler/_helpers/expansion.rs | 18 ++- .../src/scheduler/_helpers/tick_retention.rs | 43 +++++-- .../src/scheduler/main/_evaluate_trigger.rs | 2 +- .../tests/scheduler/ticks/test_retention.rs | 119 +++++++++++++++++- .../tests/scheduler/ticks/test_types.rs | 7 ++ 6 files changed, 180 insertions(+), 15 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json index 73d3ff4472..c7346bf44b 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json +++ b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json @@ -1,5 +1,5 @@ { - "description": "A backlog of covered ticks ages past the tick horizon. One cleanup pass stops at its batch cap and the next pass finishes; the requested tick and the latest covered tick stay; the operator then resets the cursor.", + "description": "A backlog of covered ticks ages past the tick horizon. One cleanup pass stops at its batch cap and the next pass finishes; the requested tick and the latest covered tick stay; the operator then resets the cursor. A yearly schedule starting years back whose latest occurrence is older than the horizon still gets exactly one latest catch-up tick and nothing older; bounded catch-up keeps the horizon.", "shift_days": 40, "backlog_ticks": 1000, "expected_first_pass": 1000, @@ -7,5 +7,7 @@ "expected_survivors": [ "covered", "requested" - ] + ], + "yearly_starts_at": "2015-01-01T00:00:00Z", + "yearly_latest_days_ago": 100 } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs index 75337f66d1..656f465ef5 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -1,6 +1,6 @@ //! Bound cron backlog work without losing omitted times or requesting an obsolete latest run. -use std::ops::Bound::{Excluded, Unbounded}; +use std::ops::Bound::{Excluded, Included, Unbounded}; use chrono::{DateTime, Utc}; @@ -56,3 +56,19 @@ pub(crate) fn expand_schedule( } Ok((due, Some(continuation))) } + +/// The newest occurrence at or before `now`, found without expanding the history before it. +pub(crate) fn latest_occurrence( + trigger: &TriggerDefinition, + now: DateTime, +) -> Result>, ControlError> { + match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => previous_cron_time(&parse_cron(expression, timezone)?, now).map(Some), + Schedule::Calendar { times, .. } => { + Ok(times.range((Unbounded, Included(now))).next_back().copied()) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs index db668918c7..c7591272e9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs @@ -5,28 +5,53 @@ use std::time::{Duration, Instant}; use chrono::{DateTime, TimeDelta, Utc}; use uuid::Uuid; +use crate::scheduler::_helpers::expansion::latest_occurrence; use crate::scheduler::constants::{ TICK_RETENTION_BATCH, TICK_RETENTION_BATCHES_PER_PASS, TICK_RETENTION_DAYS, TICK_RETENTION_INTERVAL_SECONDS, }; use crate::scheduler::main::check_lease::check_lease; -use crate::scheduler::models::SchedulerLease; +use crate::scheduler::models::{CatchUp, DueTrigger, SchedulerLease}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; +use crate::store::models::{Change, ControlStore, ControlTransaction, Parameter, TransactionMode}; -/// Evaluation never starts before the horizon, so a cursor reset cannot resurrect deleted ticks. -pub(crate) fn evaluation_start( - (last_tick_for, started_at): (Option>, DateTime), +/// Evaluation starts at the horizon, except that an unticked `latest` occurrence keeps its one tick. +pub(crate) async fn evaluation_start( + transaction: ControlTransaction, + trigger: &DueTrigger, now: DateTime, -) -> Result, ControlError> { - let start = match last_tick_for { +) -> Result<(ControlTransaction, DateTime), ControlError> { + let start = match trigger.last_tick_for { Some(last) => last, - None => started_at + None => trigger + .started_at .checked_sub_signed(TimeDelta::microseconds(1)) .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, }; - Ok(start.max(now - TimeDelta::days(TICK_RETENTION_DAYS))) + let horizon = now - TimeDelta::days(TICK_RETENTION_DAYS); + if start >= horizon { + return Ok((transaction, start)); + } + if trigger.definition_json.catch_up == CatchUp::Latest + && let Some(latest) = latest_occurrence(&trigger.definition_json, now)? + && latest > start + && latest < horizon + { + let (transaction, rows) = query( + transaction, + "SELECT tick_id FROM sqb_ticks WHERE trigger_id=$1 AND scheduled_for>=$2 LIMIT 1", + vec![Parameter::Id(trigger.trigger_id), Parameter::Time(latest)], + ) + .await?; + let from = if rows.is_empty() { + latest - TimeDelta::seconds(1) + } else { + horizon + }; + return Ok((transaction, from)); + } + Ok((transaction, horizon)) } /// Run the leader's tick cleanup at most once per interval; a capped pass runs again next cycle. diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs index b6fe95739c..eef5f67f87 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -42,7 +42,7 @@ pub(crate) async fn evaluate_trigger( return Ok(None); } let now = transaction.now; - let from = evaluation_start((trigger.last_tick_for, trigger.started_at), now)?; + let (transaction, from) = evaluation_start(transaction, &trigger, now).await?; let (due_times, continuation) = expand_schedule(&trigger.definition_json, from, now)?; let (transaction, rows) = query(transaction, "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs index 0ead6f6bf1..cf1a752008 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -1,12 +1,14 @@ //! Expired skipped/covered ticks go; a cursor reset never recreates history older than the horizon. -use chrono::{TimeDelta, Utc}; +use chrono::{Datelike, DurationRound, TimeDelta, Utc}; +use serde_json::json; use uuid::Uuid; use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::list_runs::list_runs; use crate::scheduler::_helpers::tick_retention::prune_ticks; use crate::scheduler::main::{ _acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger, list_ticks::list_ticks, @@ -15,7 +17,9 @@ use crate::scheduler::models::TickOutcome; use crate::scheduler::tests::contracts::ticks::helpers::{ append_covered_tick, calendar_time, shift_schedule, snapshot, }; -use crate::scheduler::tests::contracts::ticks::test_types::TickRetentionTestCase; +use crate::scheduler::tests::contracts::ticks::test_types::{ + LatestCatchUpTestCase, TickRetentionTestCase, +}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; @@ -167,3 +171,114 @@ fn given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_ }); } } + +#[test] +fn given_an_old_yearly_schedule_when_first_evaluated_then_latest_catch_up_ticks_only_its_latest_occurrence() + { + let test_cases = [ + LatestCatchUpTestCase { + description: "SQLite latest catch-up ticks an occurrence older than the horizon", + backend: Backend::Sqlite, + catch_up: json!("latest"), + expected_ticks: 1, + }, + LatestCatchUpTestCase { + description: "PostgreSQL latest catch-up ticks an occurrence older than the horizon", + backend: Backend::Postgres, + catch_up: json!("latest"), + expected_ticks: 1, + }, + LatestCatchUpTestCase { + description: "bounded catch-up keeps the horizon clamp", + backend: Backend::Sqlite, + catch_up: json!({"all": {"max_runs": 5}}), + expected_ticks: 0, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/tick_retention.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("tick fixture")) + .expect("tick case"); + // A yearly date (day 28 at most) well past the horizon, so its latest occurrence is older than it. + let days_ago = case["yearly_latest_days_ago"].as_i64().expect("days ago"); + let past = Utc::now() - TimeDelta::days(days_ago); + let latest = past + .with_day(past.day().min(28)) + .expect("day every year has") + .duration_trunc(TimeDelta::days(1)) + .expect("midnight"); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({"yearly_orders": {"description": "Yearly orders", "job_key": "orders", + "starts_at": case["yearly_starts_at"], "catch_up": test_case.catch_up, + "schedule": {"kind": "cron", "expression": format!("0 0 {} {} *", latest.day(), latest.month()), "timezone": "UTC"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + for _ in 0..2 { + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("catch up") + .expect("trigger"); + } + let ticks: Vec<_> = list_ticks(store, trigger_id, 10) + .await + .expect("ticks") + .into_iter() + .map(|tick| (tick.scheduled_for, tick.outcome)) + .collect(); + assert_eq!( + ticks.len(), + test_case.expected_ticks, + "{}", + test_case.description + ); + assert!( + ticks + .iter() + .all(|tick| *tick == (latest, TickOutcome::Requested)), + "only the latest occurrence is ticked: {}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("runs"); + assert_eq!( + runs.len(), + test_case.expected_ticks, + "{}", + test_case.description + ); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger") + .remove(0); + assert_eq!( + trigger.next_due_at, + Some(latest.with_year(latest.year() + 1).expect("next year")), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 03615e2ad3..3d4fb7d5d2 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -41,3 +41,10 @@ pub(crate) struct TickRetentionTestCase { pub(crate) backend: Backend, pub(crate) expected_survivors: usize, } + +pub(crate) struct LatestCatchUpTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) catch_up: serde_json::Value, + pub(crate) expected_ticks: usize, +} From a9e0b4fea665bbc66c70c641fdffbf649d98aeb3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:06:17 +0000 Subject: [PATCH 133/138] fix: start cron searches on whole seconds so first ticks land on their occurrence A never-ticked trigger starts searching from its sub-second start time, which leaked the fraction into the first tick (00:00:00.999999). Forward and backward cron searches now start from the whole second. A cursor left just past an occurrence still resumes at the next occurrence, so state written before this fix does not re-tick. --- .../orchestration/first_tick_precision.json | 7 + .../src/scheduler/main/_next_cron_time.rs | 9 +- .../src/scheduler/main/_previous_cron_time.rs | 11 +- .../src/scheduler/tests/test_cron.rs | 25 +++- .../tests/scheduler/ticks/helpers.rs | 27 ++++ .../tests/scheduler/ticks/test_precision.rs | 131 ++++++++++++++++++ .../tests/scheduler/ticks/test_ticks.rs | 2 + .../tests/scheduler/ticks/test_types.rs | 8 ++ 8 files changed, 212 insertions(+), 8 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json b/benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json new file mode 100644 index 0000000000..fd47f3312e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json @@ -0,0 +1,7 @@ +{ + "description": "A minutely cron trigger whose recorded start has a sub-second fraction ticks on whole minutes only. A cursor left just past its occurrence by an earlier build resumes without ticking that occurrence again.", + "expression": "* * * * *", + "start_minutes_ago": 5, + "start_fraction_microseconds": 123456, + "legacy_cursor_fraction_microseconds": 999999 +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs index 90c51039cc..b06ca823e1 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs @@ -1,17 +1,20 @@ //! Calculate the first distinct UTC tick after a persisted cursor. -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Timelike, Utc}; use crate::scheduler::_helpers::calendar::{resolve_wall_time, search_start}; use crate::scheduler::models::CronSchedule; use crate::store::errors::ControlError; -/// Ambiguities select the later fold; missing times move to the end of the actual IANA gap. +/// Later fold for ambiguities, IANA gap end for missing times, searching from the cursor's whole second. pub(in crate::scheduler) fn next_cron_time( schedule: &CronSchedule, after: DateTime, ) -> Result, ControlError> { - let mut cursor = search_start(schedule.timezone, after).and_utc(); + let mut cursor = search_start(schedule.timezone, after) + .with_nanosecond(0) + .ok_or_else(|| ControlError::Invalid("cron cursor is out of range".into()))? + .and_utc(); loop { let candidate = schedule .expression diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs index 3297e839ba..fdc29f8c28 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs @@ -1,17 +1,22 @@ //! Locate the latest due tick without expanding an unbounded catch-up backlog. -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Timelike, Utc}; use crate::scheduler::_helpers::calendar::resolve_wall_time; use crate::scheduler::models::CronSchedule; use crate::store::errors::ControlError; -/// Fold and gap mapping is identical to forward expansion, including the later ambiguous instant. +/// Forward expansion's fold and gap mapping, searching from `at`'s whole second so results stay on occurrences. pub(in crate::scheduler) fn previous_cron_time( schedule: &CronSchedule, at: DateTime, ) -> Result, ControlError> { - let mut cursor = at.with_timezone(&schedule.timezone).naive_local().and_utc(); + let mut cursor = at + .with_timezone(&schedule.timezone) + .naive_local() + .with_nanosecond(0) + .ok_or_else(|| ControlError::Invalid("cron search time is out of range".into()))? + .and_utc(); let mut inclusive = true; loop { let candidate = schedule diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs index fac0537e06..c7d258cc3d 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -78,6 +78,20 @@ fn given_cron_and_iana_time_zone_when_crossing_dst_then_utc_ticks_follow_the_exp after: "2026-03-08T06:17:00Z", expected_next: "2026-03-09T06:17:00Z", }, + CronTestCase { + description: "a first tick after a sub-second trigger start lands on the whole second", + expression: "0 0 * * *", + timezone: "UTC", + after: "2026-07-26T23:59:59.999999Z", + expected_next: "2026-07-27T00:00:00Z", + }, + CronTestCase { + description: "a sub-second cursor past its occurrence does not repeat that occurrence", + expression: "0 0 * * *", + timezone: "UTC", + after: "2026-07-27T00:00:00.999999Z", + expected_next: "2026-07-28T00:00:00Z", + }, ]; for test_case in test_cases { let schedule = @@ -87,7 +101,7 @@ fn given_cron_and_iana_time_zone_when_crossing_dst_then_utc_ticks_follow_the_exp .to_utc(); let observed = next_cron_time(&schedule, after).expect("next cron time exists"); assert_eq!( - observed.to_rfc3339_opts(SecondsFormat::Secs, true), + observed.to_rfc3339_opts(SecondsFormat::AutoSi, true), test_case.expected_next, "{}", test_case.description @@ -141,6 +155,13 @@ fn given_a_latest_tick_search_when_crossing_folds_and_gaps_then_it_agrees_with_f at: "2011-12-30T10:00:00Z", expected_previous: "2011-12-30T10:00:00Z", }, + PreviousCronTestCase { + description: "a sub-second search time finds the whole-second occurrence", + expression: "0 0 * * *", + timezone: "UTC", + at: "2026-07-27T00:00:00.5Z", + expected_previous: "2026-07-27T00:00:00Z", + }, ]; for test_case in test_cases { let cron = parse_cron(test_case.expression, test_case.timezone).expect("valid cron"); @@ -149,7 +170,7 @@ fn given_a_latest_tick_search_when_crossing_folds_and_gaps_then_it_agrees_with_f .to_utc(); let previous = previous_cron_time(&cron, at).expect("previous scheduled instant"); assert_eq!( - previous.to_rfc3339_opts(SecondsFormat::Secs, true), + previous.to_rfc3339_opts(SecondsFormat::AutoSi, true), test_case.expected_previous, "{}", test_case.description diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 705fee95e0..eb26f231bb 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -413,3 +413,30 @@ pub(crate) async fn append_covered_tick( .expect("append covered tick"); commit(transaction).await.expect("commit covered tick"); } + +/// Rewrite one trigger-state column the way an earlier build or a past start would have left it. +pub(crate) async fn set_trigger_state( + store: &ControlStore, + trigger_id: Uuid, + sql: &str, + time: DateTime, +) { + let transaction = begin(store, TransactionMode::Write).await.expect("state"); + let (transaction, _) = query( + transaction, + sql, + vec![Parameter::Time(time), Parameter::Id(trigger_id)], + ) + .await + .expect("rewrite trigger state"); + commit(transaction).await.expect("commit state"); +} + +pub(crate) async fn tick_times(store: &ControlStore, trigger_id: Uuid) -> Vec> { + list_ticks(store, trigger_id, 1000) + .await + .expect("ticks") + .into_iter() + .map(|tick| tick.scheduled_for) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs new file mode 100644 index 0000000000..7aa8f12c3d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs @@ -0,0 +1,131 @@ +//! A sub-second trigger start or cursor never moves a cron tick off its whole-second occurrence. + +use std::collections::BTreeSet; + +use chrono::{DateTime, DurationRound, TimeDelta, Timelike, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; +use crate::scheduler::tests::contracts::ticks::helpers::{set_trigger_state, snapshot, tick_times}; +use crate::scheduler::tests::contracts::ticks::test_types::FirstTickPrecisionTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_sub_second_trigger_start_or_cursor_when_evaluated_then_cron_ticks_land_on_whole_seconds_once() + { + let test_cases = [ + FirstTickPrecisionTestCase { + description: "SQLite first and resumed cron ticks", + backend: Backend::Sqlite, + expected_first_tick_minutes_after_start: 1, + expected_fractional_ticks: 0, + expected_repeated_occurrences: 0, + }, + FirstTickPrecisionTestCase { + description: "PostgreSQL first and resumed cron ticks", + backend: Backend::Postgres, + expected_first_tick_minutes_after_start: 1, + expected_fractional_ticks: 0, + expected_repeated_occurrences: 0, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json", + ); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("precision fixture")) + .expect("precision case"); + let number = |key: &str| case[key].as_i64().expect("fixture number"); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({"minutely_orders": {"description": "Minutely orders", "job_key": "orders", + "starts_at": "2026-01-01T00:00:00Z", "catch_up": {"all": {"max_runs": 20}}, + "schedule": {"kind": "cron", "expression": case["expression"], "timezone": "UTC"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + // Publication records a sub-second start; place it a few minutes back so ticks are due. + let start_minute = Utc::now() + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute") + - TimeDelta::minutes(number("start_minutes_ago")); + let started = + start_minute + TimeDelta::microseconds(number("start_fraction_microseconds")); + set_trigger_state(store, trigger_id, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=NULL, next_due_at=NULL WHERE trigger_id=$2", + started).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("first evaluation") + .expect("trigger"); + let first = tick_times(store, trigger_id).await; + assert_eq!( + first.iter().min(), + Some( + &(start_minute + + TimeDelta::minutes(test_case.expected_first_tick_minutes_after_start)) + ), + "{}", + test_case.description + ); + // A cursor written by an earlier build sits just past its occurrence; resuming from it + // must not tick that occurrence again at the whole second. + let latest: DateTime = *first.iter().max().expect("latest tick"); + let legacy = + latest + TimeDelta::microseconds(number("legacy_cursor_fraction_microseconds")); + set_trigger_state( + store, + trigger_id, + "UPDATE sqb_trigger_state SET last_tick_for=$1 WHERE trigger_id=$2", + legacy, + ) + .await; + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("resumed evaluation") + .expect("trigger"); + let resumed = tick_times(store, trigger_id).await; + let seconds: BTreeSet = resumed.iter().map(DateTime::timestamp).collect(); + assert_eq!( + resumed.len() - seconds.len(), + test_case.expected_repeated_occurrences, + "{}: {resumed:?}", + test_case.description + ); + assert_eq!( + resumed + .iter() + .filter(|time| time.second() != 0 || time.nanosecond() != 0) + .count(), + test_case.expected_fractional_ticks, + "{}: {resumed:?}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index cd677556bf..4daacc6f66 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -4,6 +4,8 @@ pub(crate) mod helpers; mod test_contract; #[path = "test_loop.rs"] mod test_loop; +#[path = "test_precision.rs"] +mod test_precision; #[path = "test_progress.rs"] mod test_progress; #[path = "test_retention.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 3d4fb7d5d2..9e52b992fa 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -48,3 +48,11 @@ pub(crate) struct LatestCatchUpTestCase { pub(crate) catch_up: serde_json::Value, pub(crate) expected_ticks: usize, } + +pub(crate) struct FirstTickPrecisionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_first_tick_minutes_after_start: i64, + pub(crate) expected_fractional_ticks: usize, + pub(crate) expected_repeated_occurrences: usize, +} From 46383c51018be34425597b3344a16ff45b813521 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:06:17 +0000 Subject: [PATCH 134/138] test: simulate scheduler faults, failovers and DST transitions Two schedulers on separate store handles run seeded random steps of crash-restarts, lease lapses, injected transaction faults and single or concurrent cycles on SQLite and PostgreSQL. After recovery every due minute has exactly one tick, one queued run and one request event, every frozen leader's write is fenced, and the database rejects a duplicate tick. The pure simulation now expands real cron schedules across both New York transitions under random crashes and backward and forward clock jumps, and compares the ticks with occurrences derived day by day from wall times. --- .../src/scheduler/tests/helpers.rs | 131 +++++++++ .../src/scheduler/tests/test_simulation.rs | 48 +++- .../src/scheduler/tests/test_types.rs | 8 + .../tests/scheduler/ticks/helpers.rs | 251 ++++++++++++++++-- .../tests/scheduler/ticks/test_faults.rs | 42 +++ .../tests/scheduler/ticks/test_ticks.rs | 2 + .../tests/scheduler/ticks/test_types.rs | 26 ++ 7 files changed, 481 insertions(+), 27 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs index 0484e35cac..2057452571 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -8,6 +8,8 @@ use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; use proptest::{prop_assert, prop_assert_eq}; use uuid::Uuid; +use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; +use crate::scheduler::_helpers::expansion::expand_schedule; use crate::scheduler::main::_plan_ticks::plan_ticks; use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickPlan, TickState}; use crate::scheduler::tests::test_types::TickTestCase; @@ -127,3 +129,132 @@ pub(super) fn simulate(catch_up: CatchUp) -> Result<(), String> { }) .map_err(|error| error.to_string()) } + +/// Reference occurrences found day by day from local wall times, independent of cursor search. +fn reference_occurrences( + timezone: chrono_tz::Tz, + wall_times: &[(u32, u32)], + (start, end): (DateTime, DateTime), +) -> BTreeSet> { + use chrono::{TimeDelta, TimeZone}; + let first_date = start.with_timezone(&timezone).date_naive() - TimeDelta::days(1); + let days = (end.with_timezone(&timezone).date_naive() - first_date).num_days(); + (0..=days) + .flat_map(|day| { + wall_times.iter().map(move |(hour, minute)| { + let wall = (first_date + TimeDelta::days(day)) + .and_hms_opt(*hour, *minute, 0) + .expect("wall time"); + // The later fold for repeated times; the first existing minute after a gap. + (0..=180) + .find_map(|offset| { + timezone + .from_local_datetime(&(wall + TimeDelta::minutes(offset))) + .latest() + }) + .expect("a local time exists within three hours") + .to_utc() + }) + }) + .filter(|instant| *instant > start && *instant <= end) + .collect() +} + +/// Real cron expansion and planning under random crashes and clock jumps, including backwards. +fn exercise_cron_history( + (trigger, timezone): (&TriggerDefinition, chrono_tz::Tz), + wall_times: &[(u32, u32)], + (start, end): (DateTime, DateTime), + history: Vec<(i64, bool)>, +) -> TestCaseResult { + use chrono::TimeDelta; + let mut last_tick_for: Option> = None; + let mut committed: BTreeSet> = BTreeSet::new(); + let mut clock = start; + // After the random history, enough uncrashed steps at the end time to drain a bounded backlog. + let steps = history + .into_iter() + .map(|(minutes, commit)| (TimeDelta::minutes(minutes), commit)) + .chain(std::iter::repeat_n((end - start, true), 500)); + for (jump, commit) in steps { + clock = (clock + jump).clamp(start, end); + let (due_times, continuation) = expand_schedule( + trigger, + last_tick_for.unwrap_or(start - TimeDelta::microseconds(1)), + clock, + ) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let plan = plan_ticks( + &TickState { + trigger_id: Uuid::from_u128(23), + enabled: true, + last_tick_for, + due_times, + continuation, + catch_up: trigger.catch_up, + overlap: Overlap::Queue, + waiting_run: None, + }, + clock, + ) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + for action in plan.actions.iter().filter(|_| commit) { + prop_assert!( + action.scheduled_for <= clock, + "a tick ran before it was due" + ); + prop_assert!( + committed.insert(action.scheduled_for), + "a committed tick was emitted twice" + ); + } + last_tick_for = [last_tick_for, plan.last_tick_for][usize::from(commit)]; + } + prop_assert_eq!( + committed, + reference_occurrences(timezone, wall_times, (start, end)), + "a due tick was lost or invented" + ); + Ok(()) +} + +/// Months spanning both 2026 New York transitions; clocks jump up to a day back or four days on. +pub(super) fn simulate_cron( + expression: &str, + wall_times: &[(u32, u32)], + catch_up: CatchUp, +) -> Result<(), String> { + let start = DateTime::parse_from_rfc3339("2026-02-25T00:00:00.25Z") + .expect("start") + .to_utc(); + let end = DateTime::parse_from_rfc3339("2026-11-10T00:00:00Z") + .expect("end") + .to_utc(); + let trigger = TriggerDefinition { + description: "Simulated orders".into(), + job_key: "orders".into(), + schedule: Schedule::Cron { + expression: expression.into(), + timezone: "America/New_York".into(), + }, + starts_at: Some(start), + default_status: TriggerStatus::default(), + catch_up, + }; + let strategy = proptest::collection::vec((-1_440_i64..5_760, any::()), 0..256); + let mut property_runner = TestRunner::new(Config { + cases: 64, + failure_persistence: None, + ..Config::default() + }); + property_runner + .run(&strategy, |history| { + exercise_cron_history( + (&trigger, chrono_tz::America::New_York), + wall_times, + (start, end), + history, + ) + }) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs index 60e0de2784..962e0b860f 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs @@ -1,8 +1,8 @@ use std::num::NonZeroUsize; use crate::scheduler::models::CatchUp; -use crate::scheduler::tests::helpers::simulate; -use crate::scheduler::tests::test_types::SimulationTestCase; +use crate::scheduler::tests::helpers::{simulate, simulate_cron}; +use crate::scheduler::tests::test_types::{CronSimulationTestCase, SimulationTestCase}; #[test] fn given_random_crashes_and_clock_jumps_when_restarting_then_every_due_tick_commits_once() { @@ -30,3 +30,47 @@ fn given_random_crashes_and_clock_jumps_when_restarting_then_every_due_tick_comm ); } } + +#[test] +fn given_dst_transitions_crashes_and_clock_jumps_when_expanding_real_cron_then_every_occurrence_ticks_once() + { + let bounded = CatchUp::All { + max_runs: NonZeroUsize::new(3).expect("nonzero test batch"), + }; + let test_cases = [ + CronSimulationTestCase { + description: "a daily time in the autumn fold ticks once, at the later instant", + expression: "30 1 * * *", + wall_times: &[(1, 30)], + catch_up: CatchUp::Latest, + expected_success: true, + }, + CronSimulationTestCase { + description: "a daily time in the spring gap ticks once, at the gap end", + expression: "30 2 * * *", + wall_times: &[(2, 30)], + catch_up: bounded, + expected_success: true, + }, + CronSimulationTestCase { + description: "several times inside the fold and gap keep their distinct instants", + expression: "*/30 1-2 * * *", + wall_times: &[(1, 0), (1, 30), (2, 0), (2, 30)], + catch_up: bounded, + expected_success: true, + }, + ]; + for test_case in test_cases { + let observed = simulate_cron( + test_case.expression, + test_case.wall_times, + test_case.catch_up, + ); + assert_eq!( + observed.is_ok(), + test_case.expected_success, + "{}: {observed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs index ab3d443e4f..07e3abfbd2 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -59,3 +59,11 @@ pub(super) struct SimulationTestCase { pub(super) catch_up: CatchUp, pub(super) expected_success: bool, } + +pub(super) struct CronSimulationTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) wall_times: &'static [(u32, u32)], + pub(super) catch_up: CatchUp, + pub(super) expected_success: bool, +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index eb26f231bb..2c08c1de44 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -1,16 +1,17 @@ //! Trigger and failure fixtures exercise the scheduler's transactional tick boundary. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::LazyLock; -use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Utc}; +use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Timelike, Utc}; use crate::declarations::main::list_triggers::list_triggers; use crate::runs::main::list_runs::list_runs; use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::main::list_ticks::list_ticks; -use crate::scheduler::models::TickOutcome; +use crate::scheduler::models::{SchedulerLease, SchedulerState, TickOutcome}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; @@ -21,7 +22,9 @@ use serde_json::{Value, json}; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; -use crate::scheduler::tests::contracts::ticks::test_types::TransactionFault; +use crate::scheduler::tests::contracts::ticks::test_types::{ + FaultAnomalies, FaultStep, TransactionFault, +}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::helpers::read_recall_events; use crate::store::tests::lease_helpers::expire_lease; @@ -123,26 +126,9 @@ pub(crate) async fn trigger_audits(store: &ControlStore) -> Vec { .collect() } -pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { - let fixture = open_test_store(backend).await?; - let store = &fixture.store; - migrate_store(store).await.expect("migrate tick store"); - let location = create_location(store, "orders", "test-admin") - .await - .expect("create location"); - publish_snapshot(store, &snapshot(location)) - .await - .expect("publish job and trigger"); - let trigger_id = list_triggers(store, location, "development") - .await - .expect("published triggers")[0] - .trigger_id; - let lease = acquire_lease(store, Uuid::now_v7()) - .await - .expect("acquire lease") - .expect("elected"); - let second = second_store(store).await; - for point in [ +/// Crash points inside the tick transaction, from the tick row to the cursor and the fence. +pub(crate) fn tick_faults() -> [TransactionFault; 7] { + [ TransactionFault { description: "tick insertion", table: "sqb_ticks", @@ -199,7 +185,29 @@ pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usi sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", }, - ] { + ] +} + +pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate tick store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish job and trigger"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("published triggers")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("elected"); + let second = second_store(store).await; + for point in tick_faults() { install_fault(store, &point).await; let failed = evaluate_trigger(store, &lease, trigger_id).await; assert!(failed.is_err(), "{} must abort", point.description); @@ -440,3 +448,196 @@ pub(crate) async fn tick_times(store: &ControlStore, trigger_id: Uuid) -> Vec usize { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut value = *state; + value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + usize::try_from((value ^ (value >> 31)) % bound).expect("choice fits usize") +} + +pub(crate) fn draw_step(state: &mut u64) -> FaultStep { + FaultStep { + restart: [None, None, None, None, Some(0), Some(1)][draw(state, 6)], + expire: draw(state, 4) == 0, + fault: draw(state, 14).min(7), + cycles: [[true, false], [false, true], [true, true]][draw(state, 3)], + } +} + +/// One process cycle; an error is that process crashing and restarting with a new holder. +async fn cycle_or_restart(store: &ControlStore, state: SchedulerState) -> (SchedulerState, usize) { + let result = scheduler_cycle(store, state).await; + let crashed = usize::from(result.is_err()); + ( + result.map_or_else(|_| SchedulerState::new(Uuid::now_v7()), |(state, _)| state), + crashed, + ) +} + +/// Seeded random faults across two schedulers, then recovery; returns contract breaches and step counts. +pub(crate) async fn fault_run( + backend: Backend, + seed: u64, + steps: usize, +) -> Option<(FaultAnomalies, BTreeMap<&'static str, usize>)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"], "overlap": "queue"}}), + ); + input.documents.insert( + "triggers".into(), + json!({"minutely_orders": {"description": "Minutely orders", "job_key": "orders", + "starts_at": "2026-01-01T00:00:00Z", "catch_up": {"all": {"max_runs": 3}}, + "schedule": {"kind": "cron", "expression": "* * * * *", "timezone": "UTC"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let started = Utc::now() + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute") + - TimeDelta::minutes(20) + + TimeDelta::milliseconds(250); + set_trigger_state(store, trigger_id, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=NULL, next_due_at=$1 WHERE trigger_id=$2", + started).await; + let second = second_store(store).await; + let handles = [store, &second]; + let mut states = [ + SchedulerState::new(Uuid::now_v7()), + SchedulerState::new(Uuid::now_v7()), + ]; + let faults: Vec = tick_faults() + .into_iter() + .chain([TransactionFault { + description: "no fault", + table: "sqb_ticks", + operation: "INSERT", + condition: "WHEN 1=0", + sqlite_body: "SELECT 1", + postgres_body: "PERFORM 1", + }]) + .collect(); + let mut choices = seed; + let mut counts: BTreeMap<&'static str, usize> = BTreeMap::new(); + let mut anomalies = FaultAnomalies::default(); + for _ in 0..steps { + let step = draw_step(&mut choices); + step.restart + .into_iter() + .for_each(|index| states[index] = SchedulerState::new(Uuid::now_v7())); + // A lapsed lease leaves every current holder frozen on an epoch that must now be fenced. + let stale: Vec = states + .iter() + .filter_map(|state| state.lease.clone()) + .filter(|_| step.expire) + .collect(); + for _ in 0..usize::from(step.expire) { + expire_lease(store).await; + } + let fault = &faults[step.fault]; + install_fault(store, fault).await; + let cycle = |index: usize| { + let (handle, mut state, cycles) = + (handles[index], states[index].clone(), step.cycles[index]); + async move { + let mut crashed = 0; + for _ in 0..usize::from(cycles) { + (state, crashed) = cycle_or_restart(handle, state).await; + } + (state, crashed) + } + }; + let ((left, left_crashed), (right, right_crashed)) = tokio::join!(cycle(0), cycle(1)); + states = [left, right]; + remove_fault(store, fault).await; + for lease in &stale { + anomalies.unfenced_stale_writes += usize::from(!matches!( + evaluate_trigger(store, lease, trigger_id).await, + Err(ControlError::Fenced) + )); + } + for (name, count) in [ + ("restarts", usize::from(step.restart.is_some())), + ("lease expiries", usize::from(step.expire)), + ("armed faults", usize::from(step.fault < 7)), + ("crashes", left_crashed + right_crashed), + ( + "concurrent cycles", + usize::from(step.cycles == [true, true]), + ), + ("frozen leases fenced", stale.len()), + ] { + *counts.entry(name).or_default() += count; + } + } + // Recovery: faults are gone and leadership lapses; one survivor drains whatever is still due. + expire_lease(store).await; + let mut survivor = SchedulerState::new(Uuid::now_v7()); + for _ in 0..20 { + survivor = scheduler_cycle(&second, survivor) + .await + .expect("recovery") + .0; + } + let settled = Utc::now(); + scheduler_cycle(&second, survivor) + .await + .expect("final recovery"); + let ticks = list_ticks(store, trigger_id, 1000).await.expect("ticks"); + let times: BTreeSet> = ticks.iter().map(|tick| tick.scheduled_for).collect(); + let first = started + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute") + + TimeDelta::minutes(1); + let expected: BTreeSet> = (0..) + .map(|minutes| first + TimeDelta::minutes(minutes)) + .take_while(|time| *time <= settled) + .collect(); + anomalies.duplicate_ticks = ticks.len() - times.len(); + anomalies.missing_ticks = expected.difference(×).count(); + anomalies.off_schedule_ticks = times + .iter() + .filter(|time| **time < first || time.second() != 0 || time.nanosecond() != 0) + .count(); + let runs: BTreeSet = list_runs(store, (location, "development"), 1000) + .await + .expect("runs") + .iter() + .map(|run| run.run_id) + .collect(); + let tick_runs: BTreeSet = ticks.iter().filter_map(|tick| tick.run_id).collect(); + anomalies.runs_without_exactly_one_tick = runs.symmetric_difference(&tick_runs).count() + + ticks.iter().filter(|tick| tick.run_id.is_none()).count(); + let requested = read_recall_events(store, 0, 10_000) + .await + .expect("events") + .iter() + .filter(|event| event["event_type"] == "run_requested") + .count(); + anomalies.request_event_mismatch = requested.abs_diff(ticks.len()); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("duplicate"); + let duplicate = query(transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, details_json) \ + VALUES ($1,$2,$3,$3,1,'skipped',NULL,NULL,$4)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Id(trigger_id), Parameter::Time(first), Parameter::Json(json!({}))]).await; + anomalies.accepted_duplicate_ticks = usize::from(duplicate.is_ok()); + counts.insert("ticks", ticks.len()); + close_store(second).await; + close_test_store(fixture).await; + Some((anomalies, counts)) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs new file mode 100644 index 0000000000..604b2dc4de --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs @@ -0,0 +1,42 @@ +//! Two schedulers under random crashes, faults, failovers and frozen leaders tick every due time once. + +use crate::scheduler::tests::contracts::ticks::helpers::fault_run; +use crate::scheduler::tests::contracts::ticks::test_types::{ + FaultAnomalies, SchedulerFaultTestCase, +}; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_schedulers_under_random_faults_failovers_and_frozen_leaders_when_recovered_then_every_due_tick_happens_once() + { + let test_cases = [ + SchedulerFaultTestCase { + description: "SQLite two-scheduler fault injection", + backend: Backend::Sqlite, + expected_anomalies: FaultAnomalies::default(), + }, + SchedulerFaultTestCase { + description: "PostgreSQL two-scheduler fault injection", + backend: Backend::Postgres, + expected_anomalies: FaultAnomalies::default(), + }, + ]; + let seeds: u64 = std::env::var("SQB_SCHEDULER_FAULT_SEEDS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(3); + for test_case in test_cases { + for seed in 0..seeds { + let Some((anomalies, counts)) = run(fault_run(test_case.backend, seed, 60)) else { + return; + }; + println!("{} seed {seed}: {counts:?}", test_case.description); + assert_eq!( + anomalies, test_case.expected_anomalies, + "{} seed {seed}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index 4daacc6f66..0268b47509 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -2,6 +2,8 @@ pub(crate) mod helpers; #[path = "test_contract.rs"] mod test_contract; +#[path = "test_faults.rs"] +mod test_faults; #[path = "test_loop.rs"] mod test_loop; #[path = "test_precision.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 9e52b992fa..bb9771d7ba 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -56,3 +56,29 @@ pub(crate) struct FirstTickPrecisionTestCase { pub(crate) expected_fractional_ticks: usize, pub(crate) expected_repeated_occurrences: usize, } + +/// Every way a randomized fault run can break the exactly-once contract; all zero when it holds. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct FaultAnomalies { + pub(crate) duplicate_ticks: usize, + pub(crate) missing_ticks: usize, + pub(crate) off_schedule_ticks: usize, + pub(crate) runs_without_exactly_one_tick: usize, + pub(crate) request_event_mismatch: usize, + pub(crate) unfenced_stale_writes: usize, + pub(crate) accepted_duplicate_ticks: usize, +} + +pub(crate) struct SchedulerFaultTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_anomalies: FaultAnomalies, +} + +/// One randomized step: who restarts, whether the lease lapses, which fault is armed, who cycles. +pub(crate) struct FaultStep { + pub(crate) restart: Option, + pub(crate) expire: bool, + pub(crate) fault: usize, + pub(crate) cycles: [bool; 2], +} From 374264cb59dbc18e0dff393db658a7fe2982a590 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:06:17 +0000 Subject: [PATCH 135/138] test: bound scheduler tick-to-queued latency and failover catch-up A real scheduler loop queues each calendar time within 1.5 s of it falling due, and after a stopped leader's lease lapses a second scheduler queues the overdue work within 1.5 s, on SQLite and PostgreSQL. --- .../tests/scheduler/ticks/helpers.rs | 42 ++++++- .../tests/scheduler/ticks/test_latency.rs | 112 ++++++++++++++++++ .../tests/scheduler/ticks/test_ticks.rs | 2 + .../tests/scheduler/ticks/test_types.rs | 8 ++ 4 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 2c08c1de44..24a880e7fc 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -11,7 +11,8 @@ use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::main::list_ticks::list_ticks; -use crate::scheduler::models::{SchedulerLease, SchedulerState, TickOutcome}; +use crate::scheduler::main::run_scheduler::run_scheduler; +use crate::scheduler::models::{SchedulerLease, SchedulerState, TickOutcome, TickRecord}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; @@ -23,7 +24,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::scheduler::tests::contracts::ticks::test_types::{ - FaultAnomalies, FaultStep, TransactionFault, + FaultAnomalies, FaultStep, IdleLauncher, TransactionFault, }; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::helpers::read_recall_events; @@ -641,3 +642,40 @@ pub(crate) async fn fault_run( close_test_store(fixture).await; Some((anomalies, counts)) } + +/// Run one real scheduler loop until `deadline`, then stop it and return the trigger's ticks. +pub(crate) async fn run_scheduler_until( + store: &ControlStore, + trigger_id: Uuid, + deadline: DateTime, +) -> Vec { + let (stop, shutdown) = tokio::sync::watch::channel(false); + let (_changed, changes) = tokio::sync::watch::channel(()); + let stopping = async { + tokio::time::sleep((deadline - Utc::now()).to_std().unwrap_or_default()).await; + stop.send(true).expect("stop scheduler"); + }; + let (finished, ()) = tokio::join!( + run_scheduler( + store, + SchedulerState::new(Uuid::now_v7()), + (shutdown, changes), + IdleLauncher + ), + stopping + ); + finished.expect("scheduler loop"); + list_ticks(store, trigger_id, 100).await.expect("ticks") +} + +pub(crate) async fn queued_times( + store: &ControlStore, + location: Uuid, +) -> BTreeMap> { + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .into_iter() + .map(|run| (run.run_id, run.queued_at)) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs new file mode 100644 index 0000000000..d643cc1d07 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -0,0 +1,112 @@ +//! Regression bounds for tick-to-queued latency and catch-up once a stopped leader's lease lapses. + +use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; +use serde_json::json; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::tests::contracts::ticks::helpers::{ + queued_times, run_scheduler_until, snapshot, +}; +use crate::scheduler::tests::contracts::ticks::test_types::SchedulerLatencyTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks_queue_within_their_latency_bounds() + { + let test_cases = [ + SchedulerLatencyTestCase { + description: "SQLite scheduler latency", + backend: Backend::Sqlite, + expected_queued_ticks: 3, + expected_max_tick_to_queued: TimeDelta::milliseconds(1_500), + expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + }, + SchedulerLatencyTestCase { + description: "PostgreSQL scheduler latency", + backend: Backend::Postgres, + expected_queued_ticks: 3, + expected_max_tick_to_queued: TimeDelta::milliseconds(1_500), + expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let base = Utc::now() + .duration_trunc(TimeDelta::seconds(1)) + .expect("second"); + let at = |seconds| base + TimeDelta::seconds(seconds); + let text = |seconds| at(seconds).to_rfc3339_opts(SecondsFormat::Secs, true); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"], "overlap": "queue"}}), + ); + input.documents.insert( + "triggers".into(), + json!({"calendar_orders": {"description": "Calendar orders", "job_key": "orders", + "schedule": {"kind": "calendar", "times": [text(2), text(3), text(4), text(8), text(9)], + "refresh_at": "2029-01-01T00:00:00Z"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let ticks = run_scheduler_until(store, trigger_id, at(6)).await; + let queued = queued_times(store, location).await; + let latencies: Vec = ticks + .iter() + .filter_map(|tick| tick.run_id.map(|run| queued[&run] - tick.scheduled_for)) + .collect(); + println!("{} tick-to-queued: {latencies:?}", test_case.description); + assert_eq!( + latencies.len(), + test_case.expected_queued_ticks, + "{}", + test_case.description + ); + assert!( + latencies + .iter() + .all(|latency| *latency <= test_case.expected_max_tick_to_queued), + "{}: {latencies:?}", + test_case.description + ); + // The first leader stopped without releasing its lease while two more times fell due. + tokio::time::sleep((at(10) - Utc::now()).to_std().unwrap_or_default()).await; + let second = second_store(store).await; + expire_lease(store).await; + let available = Utc::now(); + let ticks = run_scheduler_until(&second, trigger_id, at(13)).await; + let queued = queued_times(store, location).await; + let latest = ticks + .iter() + .max_by_key(|tick| tick.scheduled_for) + .and_then(|tick| tick.run_id) + .expect("latest overdue time requested"); + let catch_up = queued[&latest] - available; + println!("{} failover catch-up: {catch_up:?}", test_case.description); + assert!( + catch_up <= test_case.expected_max_failover_catch_up, + "{}: {catch_up:?}", + test_case.description + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index 0268b47509..ca93c4cadf 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -4,6 +4,8 @@ pub(crate) mod helpers; mod test_contract; #[path = "test_faults.rs"] mod test_faults; +#[path = "test_latency.rs"] +mod test_latency; #[path = "test_loop.rs"] mod test_loop; #[path = "test_precision.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index bb9771d7ba..9f032b1840 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -82,3 +82,11 @@ pub(crate) struct FaultStep { pub(crate) fault: usize, pub(crate) cycles: [bool; 2], } + +pub(crate) struct SchedulerLatencyTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_queued_ticks: usize, + pub(crate) expected_max_tick_to_queued: chrono::TimeDelta, + pub(crate) expected_max_failover_catch_up: chrono::TimeDelta, +} From 8637ea5678f3f925bf63cbd7ba7fb384e67244f2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:28:50 +0000 Subject: [PATCH 136/138] test: tighten scheduler latency bounds to 500 ms Measured tick-to-queued and post-lease catch-up stay under 20 ms. A scheduler that stops waking at the next due time and falls back to its one-second poll queues ticks 0.7-0.9 s late; 1.5 s missed that, 500 ms catches it. --- .../tests/scheduler/ticks/test_latency.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index d643cc1d07..fec51960c2 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -24,15 +24,15 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks description: "SQLite scheduler latency", backend: Backend::Sqlite, expected_queued_ticks: 3, - expected_max_tick_to_queued: TimeDelta::milliseconds(1_500), - expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + expected_max_tick_to_queued: TimeDelta::milliseconds(500), + expected_max_failover_catch_up: TimeDelta::milliseconds(500), }, SchedulerLatencyTestCase { description: "PostgreSQL scheduler latency", backend: Backend::Postgres, expected_queued_ticks: 3, - expected_max_tick_to_queued: TimeDelta::milliseconds(1_500), - expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + expected_max_tick_to_queued: TimeDelta::milliseconds(500), + expected_max_failover_catch_up: TimeDelta::milliseconds(500), }, ]; for test_case in test_cases { From 5bbe6429d2d8d1cbccdc7a7e0b52acf20d962670 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:52:04 +0000 Subject: [PATCH 137/138] test: hold every scheduler crash point and a waiting standby in CI The randomized two-scheduler run now drains a two-hour backlog, so armed faults land while ticks are still being written, and it asserts that every crash point in the tick transaction fires across the seeds run (five by default). The failover check starts the second scheduler before the old lease lapses, so a standby that waits too long before retrying leadership fails; its budget is 1.5 s to allow the one-second retry. --- .../tests/scheduler/ticks/helpers.rs | 5 ++-- .../tests/scheduler/ticks/test_faults.rs | 25 ++++++++++++++-- .../tests/scheduler/ticks/test_latency.rs | 29 ++++++++++++++----- .../tests/scheduler/ticks/test_types.rs | 2 ++ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 24a880e7fc..7c27fe0dd1 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -509,7 +509,7 @@ pub(crate) async fn fault_run( let started = Utc::now() .duration_trunc(TimeDelta::minutes(1)) .expect("minute") - - TimeDelta::minutes(20) + - TimeDelta::minutes(120) + TimeDelta::milliseconds(250); set_trigger_state(store, trigger_id, "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=NULL, next_due_at=$1 WHERE trigger_id=$2", @@ -570,6 +570,7 @@ pub(crate) async fn fault_run( Err(ControlError::Fenced) )); } + *counts.entry(fault.description).or_default() += left_crashed + right_crashed; for (name, count) in [ ("restarts", usize::from(step.restart.is_some())), ("lease expiries", usize::from(step.expire)), @@ -587,7 +588,7 @@ pub(crate) async fn fault_run( // Recovery: faults are gone and leadership lapses; one survivor drains whatever is still due. expire_lease(store).await; let mut survivor = SchedulerState::new(Uuid::now_v7()); - for _ in 0..20 { + for _ in 0..60 { survivor = scheduler_cycle(&second, survivor) .await .expect("recovery") diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs index 604b2dc4de..9dd99aa3ff 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs @@ -1,6 +1,8 @@ //! Two schedulers under random crashes, faults, failovers and frozen leaders tick every due time once. -use crate::scheduler::tests::contracts::ticks::helpers::fault_run; +use std::collections::BTreeMap; + +use crate::scheduler::tests::contracts::ticks::helpers::{fault_run, tick_faults}; use crate::scheduler::tests::contracts::ticks::test_types::{ FaultAnomalies, SchedulerFaultTestCase, }; @@ -15,18 +17,21 @@ fn given_two_schedulers_under_random_faults_failovers_and_frozen_leaders_when_re description: "SQLite two-scheduler fault injection", backend: Backend::Sqlite, expected_anomalies: FaultAnomalies::default(), + expected_points_without_a_crash: 0, }, SchedulerFaultTestCase { description: "PostgreSQL two-scheduler fault injection", backend: Backend::Postgres, expected_anomalies: FaultAnomalies::default(), + expected_points_without_a_crash: 0, }, ]; let seeds: u64 = std::env::var("SQB_SCHEDULER_FAULT_SEEDS") .ok() .and_then(|value| value.parse().ok()) - .unwrap_or(3); + .unwrap_or(5); for test_case in test_cases { + let mut crashes: BTreeMap<&'static str, usize> = BTreeMap::new(); for seed in 0..seeds { let Some((anomalies, counts)) = run(fault_run(test_case.backend, seed, 60)) else { return; @@ -37,6 +42,22 @@ fn given_two_schedulers_under_random_faults_failovers_and_frozen_leaders_when_re "{} seed {seed}", test_case.description ); + for (point, count) in counts { + *crashes.entry(point).or_default() += count; + } } + // Every crash point inside the tick transaction must actually fire across the seeds run. + let unexercised: Vec<&str> = tick_faults() + .iter() + .map(|fault| fault.description) + .filter(|point| crashes.get(point).copied().unwrap_or_default() == 0) + .collect(); + println!("{} crashes per point: {crashes:?}", test_case.description); + assert_eq!( + unexercised.len(), + test_case.expected_points_without_a_crash, + "{}: {unexercised:?}", + test_case.description + ); } } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs index fec51960c2..a1d6a9db73 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -1,4 +1,4 @@ -//! Regression bounds for tick-to-queued latency and catch-up once a stopped leader's lease lapses. +//! Regression bounds for tick-to-queued latency and a waiting standby's catch-up after a lease lapses. use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; use serde_json::json; @@ -24,15 +24,17 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks description: "SQLite scheduler latency", backend: Backend::Sqlite, expected_queued_ticks: 3, + expected_ticks_after_failover: 5, expected_max_tick_to_queued: TimeDelta::milliseconds(500), - expected_max_failover_catch_up: TimeDelta::milliseconds(500), + expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), }, SchedulerLatencyTestCase { description: "PostgreSQL scheduler latency", backend: Backend::Postgres, expected_queued_ticks: 3, + expected_ticks_after_failover: 5, expected_max_tick_to_queued: TimeDelta::milliseconds(500), - expected_max_failover_catch_up: TimeDelta::milliseconds(500), + expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), }, ]; for test_case in test_cases { @@ -87,17 +89,28 @@ fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks test_case.description ); // The first leader stopped without releasing its lease while two more times fell due. + // A standby is already polling when that lease lapses, so its own retry wait counts. tokio::time::sleep((at(10) - Utc::now()).to_std().unwrap_or_default()).await; let second = second_store(store).await; - expire_lease(store).await; - let available = Utc::now(); - let ticks = run_scheduler_until(&second, trigger_id, at(13)).await; + let lapse = async { + tokio::time::sleep((at(11) - Utc::now()).to_std().unwrap_or_default()).await; + expire_lease(store).await; + Utc::now() + }; + let (ticks, available) = + tokio::join!(run_scheduler_until(&second, trigger_id, at(14)), lapse); let queued = queued_times(store, location).await; + assert_eq!( + ticks.len(), + test_case.expected_ticks_after_failover, + "{}: the standby ticks both overdue times", + test_case.description + ); let latest = ticks .iter() - .max_by_key(|tick| tick.scheduled_for) + .find(|tick| tick.scheduled_for == at(9)) .and_then(|tick| tick.run_id) - .expect("latest overdue time requested"); + .expect("the latest overdue time is requested"); let catch_up = queued[&latest] - available; println!("{} failover catch-up: {catch_up:?}", test_case.description); assert!( diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 9f032b1840..46f46eba88 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -73,6 +73,7 @@ pub(crate) struct SchedulerFaultTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, pub(crate) expected_anomalies: FaultAnomalies, + pub(crate) expected_points_without_a_crash: usize, } /// One randomized step: who restarts, whether the lease lapses, which fault is armed, who cycles. @@ -87,6 +88,7 @@ pub(crate) struct SchedulerLatencyTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, pub(crate) expected_queued_ticks: usize, + pub(crate) expected_ticks_after_failover: usize, pub(crate) expected_max_tick_to_queued: chrono::TimeDelta, pub(crate) expected_max_failover_catch_up: chrono::TimeDelta, } From 31ebba4a6299269bcedce96277c9d28bf6a2c5ed Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:52:15 +0000 Subject: [PATCH 138/138] test: prove scheduler upgrade in place and recovery from killed processes A store left by the build before trigger progress, with that build's fractional ticks and cursor and its lease still live, upgrades in place on SQLite and PostgreSQL. The upgraded scheduler waits for the old lease, resumes every later minute once, fences the old holder, and an operator repair succeeds afterwards. Installed sqb scheduler processes are SIGKILLed at seeded random moments while two of them drain a backlog; a survivor then finishes it, and every due minute of every trigger has exactly one tick, one queued run and one request event. --- .../scheduler_crash_recovery.json | 9 + .../orchestration/scheduler_upgrade.json | 6 + .../tests/scheduler/ticks/test_ticks.rs | 2 + .../tests/scheduler/ticks/test_types.rs | 9 + .../tests/scheduler/ticks/test_upgrade.rs | 218 ++++++++++++++++++ .../publication/_test_types.py | 8 + .../compiled_project/publication/helpers.py | 156 +++++++++++++ .../publication/test_scheduler_recovery.py | 50 ++++ 8 files changed, 458 insertions(+) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/scheduler_upgrade.json create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/ticks/test_upgrade.rs create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_scheduler_recovery.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json new file mode 100644 index 0000000000..f3bf7e442b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json @@ -0,0 +1,9 @@ +{ + "description": "Three minutely cron jobs with queue overlap and bounded all catch-up start ten hours back. Two installed schedulers drain the backlog while being SIGKILLed at seeded random moments; every lapse of a killed leader's lease is simulated. After a final uninterrupted scheduler, every due minute of every trigger has exactly one tick, one queued run and one request event.", + "jobs": ["orders", "customers", "inventory"], + "backlog_minutes": 600, + "max_catch_up": 1, + "rounds": 16, + "seed": 912, + "max_kill_delay_ms": 600 +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/scheduler_upgrade.json b/benchmarks/duckshop/edge_cases/orchestration/scheduler_upgrade.json new file mode 100644 index 0000000000..164a29d356 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/scheduler_upgrade.json @@ -0,0 +1,6 @@ +{ + "description": "The pre-progress build ticks a bounded batch of a minutely cron trigger at a sub-second fraction past each minute, then stops with its lease still live. The store is upgraded in place; the new scheduler waits for the old lease, then resumes from the fractional cursor and ticks each later minute once.", + "backlog_minutes": 15, + "old_build_ticks": 6, + "old_build_fraction_microseconds": 999999 +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs index ca93c4cadf..bfe8e66578 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -18,3 +18,5 @@ mod test_retention; mod test_triggers; #[path = "test_types.rs"] mod test_types; +#[path = "test_upgrade.rs"] +mod test_upgrade; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs index 46f46eba88..962da69e0a 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -92,3 +92,12 @@ pub(crate) struct SchedulerLatencyTestCase { pub(crate) expected_max_tick_to_queued: chrono::TimeDelta, pub(crate) expected_max_failover_catch_up: chrono::TimeDelta, } + +pub(crate) struct SchedulerUpgradeTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_follower_ticks: usize, + pub(crate) expected_duplicate_minutes: usize, + pub(crate) expected_missing_minutes: usize, + pub(crate) expected_runs_per_tick: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_upgrade.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_upgrade.rs new file mode 100644 index 0000000000..ecc80e116a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_upgrade.rs @@ -0,0 +1,218 @@ +//! A store written by the pre-progress build upgrades in place and its scheduler resumes exactly once. + +use std::collections::BTreeSet; + +use chrono::{DateTime, DurationRound, TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::rebuild_projections::rebuild_projections; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::main::{ + _acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger, + _scheduler_cycle::scheduler_cycle, list_ticks::list_ticks, +}; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::ticks::helpers::{set_trigger_state, snapshot}; +use crate::scheduler::tests::contracts::ticks::test_types::SchedulerUpgradeTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{ + close_test_store, open_test_store, reconstruct_fingerprint_predecessor, run, +}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_store_left_by_the_pre_progress_build_when_upgraded_then_the_scheduler_waits_for_the_old_lease_and_resumes_every_minute_once() + { + let test_cases = [ + SchedulerUpgradeTestCase { + description: "SQLite in-place scheduler upgrade", + backend: Backend::Sqlite, + expected_follower_ticks: 0, + expected_duplicate_minutes: 0, + expected_missing_minutes: 0, + expected_runs_per_tick: 1, + }, + SchedulerUpgradeTestCase { + description: "PostgreSQL in-place scheduler upgrade", + backend: Backend::Postgres, + expected_follower_ticks: 0, + expected_duplicate_minutes: 0, + expected_missing_minutes: 0, + expected_runs_per_tick: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/scheduler_upgrade.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("upgrade fixture")) + .expect("upgrade case"); + let number = |key: &str| case[key].as_i64().expect("fixture number"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"], "overlap": "queue"}}), + ); + input.documents.insert( + "triggers".into(), + json!({"minutely_orders": {"description": "Minutely orders", "job_key": "orders", + "starts_at": "2026-01-01T00:00:00Z", "catch_up": {"all": {"max_runs": number("old_build_ticks")}}, + "schedule": {"kind": "cron", "expression": "* * * * *", "timezone": "UTC"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let start = Utc::now() + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute") + - TimeDelta::minutes(number("backlog_minutes")); + set_trigger_state(store, trigger_id, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=NULL, next_due_at=$1 WHERE trigger_id=$2", + start).await; + // The old build ticks one bounded batch, then the server stops for the upgrade. + let old = acquire_lease(store, Uuid::now_v7()) + .await + .expect("old election") + .expect("old lease"); + evaluate_trigger(store, &old, trigger_id) + .await + .expect("old build ticks") + .expect("trigger"); + // That build ticked and recorded cursors a fraction past each whole minute. + let fraction = TimeDelta::microseconds(number("old_build_fraction_microseconds")); + let old_ticks = list_ticks(store, trigger_id, 1000) + .await + .expect("old ticks"); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("old precision"); + for tick in &old_ticks { + (transaction, _) = query( + transaction, + "UPDATE sqb_ticks SET scheduled_for=$1 WHERE tick_id=$2", + vec![ + Parameter::Time(tick.scheduled_for + fraction), + Parameter::Id(tick.tick_id), + ], + ) + .await + .expect("old tick"); + } + let old_cursor = old_ticks + .iter() + .map(|tick| tick.scheduled_for) + .max() + .expect("old cursor") + + fraction; + (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET last_tick_for=$1 WHERE trigger_id=$2", + vec![Parameter::Time(old_cursor), Parameter::Id(trigger_id)], + ) + .await + .expect("old cursor"); + commit(transaction).await.expect("commit old precision"); + reconstruct_fingerprint_predecessor(store).await; + migrate_store(store).await.expect("in-place upgrade"); + // The old holder's lease is still live: the upgraded scheduler must not take over. + let (upgraded, follower) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("follower cycle"); + assert!(!follower.leader, "{}", test_case.description); + assert_eq!( + follower.ticks, test_case.expected_follower_ticks, + "{}", + test_case.description + ); + expire_lease(store).await; + let mut state = upgraded; + for _ in 0..usize::try_from(number("backlog_minutes")).expect("minutes") { + state = scheduler_cycle(store, state).await.expect("resume").0; + } + assert!( + matches!( + evaluate_trigger(store, &old, trigger_id).await, + Err(ControlError::Fenced) + ), + "{}: the old build is fenced after takeover", + test_case.description + ); + let settled = Utc::now() + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute"); + let ticks = list_ticks(store, trigger_id, 1000).await.expect("ticks"); + let minutes: Vec> = ticks + .iter() + .map(|tick| { + tick.scheduled_for + .duration_trunc(TimeDelta::seconds(1)) + .expect("second") + }) + .collect(); + let distinct: BTreeSet> = minutes.iter().copied().collect(); + let expected: BTreeSet> = (0..) + .map(|offset| start + TimeDelta::minutes(offset)) + .take_while(|minute| *minute <= settled - TimeDelta::minutes(1)) + .collect(); + assert_eq!( + minutes.len() - distinct.len(), + test_case.expected_duplicate_minutes, + "{}: {minutes:?}", + test_case.description + ); + assert_eq!( + expected.difference(&distinct).count(), + test_case.expected_missing_minutes, + "{}: {minutes:?}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("runs"); + assert_eq!( + runs.len(), + ticks.len() * test_case.expected_runs_per_tick, + "{}", + test_case.description + ); + // The upgraded scheduler recorded progress, so an operator repair now succeeds. + let cursor = list_triggers(store, location, "development") + .await + .expect("cursor")[0] + .last_tick_for; + rebuild_projections(store) + .await + .expect("repair after upgrade"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("repaired")[0] + .last_tick_for, + cursor, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index f3e7050e53..c9866865b7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -317,3 +317,11 @@ class CliCleanupTestCase: description: str server: str expected_exit: int + + +@dataclass(frozen=True) +class SchedulerCrashTestCase: + description: str + fixture: str + expected_anomalies: dict[str, int] + expected_minimum_kills: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 3399cd9cf2..28fb6f4797 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -3,8 +3,10 @@ from __future__ import annotations import hashlib +import itertools import json import os +import random import secrets import signal import sqlite3 @@ -1065,3 +1067,157 @@ def statistics_relations(directory: Path) -> tuple[str, ...]: "SELECT relation_name FROM sqb_warehouse_statistics ORDER BY observed_at" ).fetchall() return tuple(name for (name,) in rows) + + +def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[str, int]: + """SIGKILL real schedulers while they drain a backlog; return breaches and the kill count.""" + project: Path = directory / "project" + start: datetime = datetime.now(UTC).replace(second=0, microsecond=0) - timedelta( + minutes=fixture["backlog_minutes"] + ) + declarations: str = "".join( + f'{name} = job("{name}", select="orders", description="Build {name}.", overlap="queue", ' + f'schedule=schedule(cron="* * * * *", timezone="UTC", catch_up="all", ' + f"max_catch_up={fixture['max_catch_up']}))\n" + for name in fixture["jobs"] + ) + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id\n" + ), + "orchestration/orders.py": "from sqlbuild.orchestration import job, schedule\n" + + declarations, + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + control_store: Path = directory / "control/.sqlbuild/control.sqlite3" + environment: dict[str, str] = os.environ | { + "SQLBUILD_SCHEDULER_TEST_STORE": f"sqlite://{control_store}" + } + for name in ("SQLBUILD_PLATFORM_WEBHOOK_URL", "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL"): + _ = environment.pop(name, None) + with snapshot_api(directory / "control") as api: + published: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "location", + "publish", + "--project-dir", + str(project), + "--location-id", + api.location_id, + "--target", + "development", + "--server", + api.url, + "--generation", + "1", + ], + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert published.returncode == 0, published.stderr + # The triggers were published hours ago, so a backlog of minutes is due when schedulers start. + with closing(sqlite3.connect(control_store)) as control, control: + _ = control.execute( + "UPDATE sqb_trigger_state SET started_at=?, next_due_at=?, last_tick_for=NULL", + (start.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),) * 2, + ) + choices: random.Random = random.Random(fixture["seed"]) + command: list[str] = [ + executable, + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + ] + kills: int = 0 + for round_index in range(fixture["rounds"]): + processes: list[subprocess.Popen[bytes]] = [ + subprocess.Popen( + command, + env=environment, + stdout=subprocess.DEVNULL, + stderr=(directory / f"scheduler-{round_index}-{index}.log").open("wb"), + ) + for index in range(2) + ] + time.sleep(choices.uniform(0.05, fixture["max_kill_delay_ms"] / 1000)) + for process in processes: + process.kill() + kills += int(process.wait(timeout=10) == -signal.SIGKILL) + # A killed holder's lease would lapse after its TTL; lapse it now instead of waiting. + with closing(sqlite3.connect(control_store, timeout=30)) as control, control: + _ = control.execute( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'" + ) + settled: datetime = datetime.now(UTC) + with closing(sqlite3.connect(control_store, timeout=30)) as control: + (interrupted,) = control.execute("SELECT count(*) FROM sqb_ticks").fetchone() + with location_process( + command, directory / "survivor.log", directory / "survivor.stdout", env=environment + ): + deadline: float = time.monotonic() + 60 + pending: int = 1 + while pending and time.monotonic() < deadline: + time.sleep(0.5) + with closing(sqlite3.connect(control_store, timeout=30)) as control: + (pending,) = control.execute( + "SELECT count(*) FROM sqb_trigger_state " + "WHERE last_tick_for IS NULL OR last_tick_for None: + fixture: dict[str, Any] = json.loads(Path(test_case.fixture).read_text()) + observed: dict[str, int] = scheduler_crash_history(tmp_path, fixture) + kills: int = observed.pop("kills") + interrupted: int = observed.pop("ticks_before_survivor") + ticks: int = observed.pop("ticks") + print(f"{test_case.description}: {kills} kills, {interrupted} of {ticks} ticks before recovery") + assert observed == test_case.expected_anomalies + assert kills >= test_case.expected_minimum_kills + assert 0 < interrupted < ticks